From db498344d6e4f569265de5a4a558e8ef73c587d8 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Tue, 20 May 2025 13:38:45 +0200 Subject: [PATCH 01/28] modify release-build for linux and macosx Signed-off-by: Pablo Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 93 ++++++++++++++--------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index b4a129bb..e12a5dff 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -129,12 +129,11 @@ jobs: image: docker:20.10.16 options: --privileged env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1+cu121'" - CIBW_BEFORE_BUILD: "pip install torch==2.3.1+cu121 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" - CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit_cuda" + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" + CIBW_BEFORE_BUILD: "pip install torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" + CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" - CIBW_BUILD: "cp39-manylinux_x86_64 cp310-manylinux_x86_64" - AIHWKIT_VERSION_SUFFIX: "+cuda121" + CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" steps: - uses: actions/checkout@v3 @@ -142,7 +141,7 @@ jobs: run: docker pull $CIBW_MANYLINUX_X86_64_IMAGE - name: Install cibuildwheel - run: python3 -m pip install cibuildwheel==2.23.3 + run: python3 -m pip install cibuildwheel - name: Build wheels run: python3 -m cibuildwheel --output-dir wheelhouse @@ -159,41 +158,41 @@ jobs: --delete \ --endpoint-url https://${{ env.COS_ENDPOINT }} - # build-and-deploy-macos: - # name: Build & Deploy wheels (macOS) - # runs-on: macos-latest - # env: - # CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.0.1'" - # CIBW_BEFORE_BUILD: "pip install torch==2.0.1 torchvision && pip install ./delocate && pip install -r requirements.txt" - # CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64" - # steps: - # - uses: actions/checkout@v3 + build-and-deploy-macos: + name: Build & Deploy wheels (macOS) + runs-on: macos-latest + env: + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.0.1'" + CIBW_BEFORE_BUILD: "pip install torch==2.0.1 torchvision && pip install ./delocate && pip install -r requirements.txt" + CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + steps: + - uses: actions/checkout@v3 - # - name: Install Homebrew deps - # run: | - # brew update - # brew install openblas + - name: Install Homebrew deps + run: | + brew update + brew install openblas - # - name: Clone delocate - # run: git clone -b aihwkit https://github.com/aihwkit-bot/delocate.git + - name: Clone delocate + run: git clone -b aihwkit https://github.com/aihwkit-bot/delocate.git - # - name: Install cibuildwheel - # run: python3 -m pip install cibuildwheel==2.8.1 + - name: Install cibuildwheel + run: python3 -m pip install cibuildwheel==2.8.1 - # - name: Build wheels - # run: python3 -m cibuildwheel --output-dir wheelhouse --platform macos + - name: Build wheels + run: python3 -m cibuildwheel --output-dir wheelhouse --platform macos - # - name: Sync wheels to IBM COS - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} - # AWS_DEFAULT_REGION: us-south-1 - # run: | - # which aws || pip install awscli --upgrade - # aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - # --acl public-read \ - # --delete \ - # --endpoint-url https://${{ env.COS_ENDPOINT }} + - name: Sync wheels to IBM COS + env: + AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-south-1 + run: | + which aws || pip install awscli --upgrade + aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ + --acl public-read \ + --delete \ + --endpoint-url https://${{ env.COS_ENDPOINT }} # build-and-deploy-windows: # name: Build & Deploy wheels (Windows) @@ -221,14 +220,14 @@ jobs: # - name: Build wheels # run: python -m cibuildwheel --output-dir wheelhouse - # - name: Sync wheels to IBM COS - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} - # AWS_DEFAULT_REGION: us-south-1 - # run: | - # which aws || pip install awscli --upgrade - # aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - # --acl public-read \ - # --delete \ - # --endpoint-url https://${{ env.COS_ENDPOINT }} + # - name: Sync wheels to IBM COS + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} + # AWS_DEFAULT_REGION: us-south-1 + # run: | + # which aws || pip install awscli --upgrade + # aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ + # --acl public-read \ + # --delete \ + # --endpoint-url https://${{ env.COS_ENDPOINT }} From 4173ff449455022cd7e02daa47a989cc8041db72 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Tue, 20 May 2025 13:53:46 +0200 Subject: [PATCH 02/28] change to pipx for macos setup Signed-off-by: Pablo Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index e12a5dff..ae046100 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -171,16 +171,16 @@ jobs: - name: Install Homebrew deps run: | brew update - brew install openblas + brew install openblas pipx - name: Clone delocate run: git clone -b aihwkit https://github.com/aihwkit-bot/delocate.git - name: Install cibuildwheel - run: python3 -m pip install cibuildwheel==2.8.1 + run: pipx install cibuildwheel==2.8.1 - name: Build wheels - run: python3 -m cibuildwheel --output-dir wheelhouse --platform macos + run: cibuildwheel --output-dir wheelhouse --platform macos - name: Sync wheels to IBM COS env: From bd3eeac3dfc4d1d70971fc16b5c83e09d355380d Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 14:53:02 +0200 Subject: [PATCH 03/28] change macosx to use pypa/cibuildwheel action and add --no-cache-dir flag to save disk space Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 43 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index ae046100..df8b3eea 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -2,8 +2,8 @@ name: release-build on: workflow_dispatch: - create: - tags: 'v**' + # create: + # tags: 'v**' permissions: id-token: write @@ -130,7 +130,7 @@ jobs: options: --privileged env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" - CIBW_BEFORE_BUILD: "pip install torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" + CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" @@ -141,7 +141,7 @@ jobs: run: docker pull $CIBW_MANYLINUX_X86_64_IMAGE - name: Install cibuildwheel - run: python3 -m pip install cibuildwheel + run: python3 -m pip --no-cache-dir install cibuildwheel - name: Build wheels run: python3 -m cibuildwheel --output-dir wheelhouse @@ -152,7 +152,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-south-1 run: | - which aws || pip install awscli --upgrade + which aws || pip install --no-cache-dir awscli --upgrade aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ --acl public-read \ --delete \ @@ -161,26 +161,23 @@ jobs: build-and-deploy-macos: name: Build & Deploy wheels (macOS) runs-on: macos-latest - env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.0.1'" - CIBW_BEFORE_BUILD: "pip install torch==2.0.1 torchvision && pip install ./delocate && pip install -r requirements.txt" - CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + strategy: + matrix: + python-version: [3.8, 3.9, 3.10] steps: - - uses: actions/checkout@v3 - - - name: Install Homebrew deps - run: | - brew update - brew install openblas pipx - - - name: Clone delocate - run: git clone -b aihwkit https://github.com/aihwkit-bot/delocate.git - - - name: Install cibuildwheel - run: pipx install cibuildwheel==2.8.1 + - uses: actions/checkout@v4 - name: Build wheels - run: cibuildwheel --output-dir wheelhouse --platform macos + uses: pypa/cibuildwheel@v2.23.3 + env: + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.0.1'" + CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.0.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + with: + # only build for macOS here + platform: macos + python-version: ${{ matrix.python-version }} + output-dir: wheelhouse - name: Sync wheels to IBM COS env: @@ -188,7 +185,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-south-1 run: | - which aws || pip install awscli --upgrade + which aws || pip install --no-cache-dir awscli --upgrade aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ --acl public-read \ --delete \ From bf034ba36cc24f96a56223f6a43dfbddf5f8d247 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 16:03:46 +0200 Subject: [PATCH 04/28] fix vars for macos build Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index df8b3eea..34332706 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -170,13 +170,10 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.0.1'" - CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.0.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" + CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" with: - # only build for macOS here - platform: macos - python-version: ${{ matrix.python-version }} output-dir: wheelhouse - name: Sync wheels to IBM COS From 287ce5696ee54f52779e8630615e04d457e1eb35 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 16:09:44 +0200 Subject: [PATCH 05/28] fix and add proper x86 arch Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 34332706..af379e45 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -172,6 +172,7 @@ jobs: env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_ARCHS: x86_64 CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" with: output-dir: wheelhouse From f92ccbf685eeed3de2a0a8c4f79886f27b946a7e Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 16:14:17 +0200 Subject: [PATCH 06/28] put proper torch version Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index af379e45..d892363f 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -170,8 +170,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" - CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" + CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" CIBW_ARCHS: x86_64 CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" with: From 083760002aa843176a728ac57ab210c30bde1bc0 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 16:16:39 +0200 Subject: [PATCH 07/28] remove unnecessary matrix Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index d892363f..0242fa39 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -161,9 +161,6 @@ jobs: build-and-deploy-macos: name: Build & Deploy wheels (macOS) runs-on: macos-latest - strategy: - matrix: - python-version: [3.8, 3.9, 3.10] steps: - uses: actions/checkout@v4 From e9c45824d80e26d78e6ce4bc7a9d2dc492e92771 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 16:24:57 +0200 Subject: [PATCH 08/28] add step for setup openblas on macos build Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 0242fa39..c2ee258b 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -164,6 +164,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Environment + run: | + brew install openblas + brew link --force openblas + - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: From 0f3311d4187c7d5846caf6f25f4b2949ff174ec4 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 16:29:27 +0200 Subject: [PATCH 09/28] change manylinux build Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index c2ee258b..16274a39 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -124,27 +124,19 @@ jobs: build-and-deploy-linux: name: Build & Deploy wheels (manylinux) runs-on: ubuntu-latest - services: - docker: - image: docker:20.10.16 - options: --privileged - env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" - CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" - CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" - CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" - CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" steps: - - uses: actions/checkout@v3 - - - name: Pull manylinux image - run: docker pull $CIBW_MANYLINUX_X86_64_IMAGE - - - name: Install cibuildwheel - run: python3 -m pip --no-cache-dir install cibuildwheel + - uses: actions/checkout@v4 - name: Build wheels - run: python3 -m cibuildwheel --output-dir wheelhouse + uses: pypa/cibuildwheel@v2.23.3 + env: + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" + CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" + CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" + CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" + with: + output-dir: wheelhouse - name: Sync wheels to IBM COS env: @@ -174,6 +166,7 @@ jobs: env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" CIBW_ARCHS: x86_64 CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" with: From 396f830b169a9e3d479aefcf57100f52342c5419 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 17:00:37 +0200 Subject: [PATCH 10/28] add clean, prune and cache tasks to help optimize disk space on runnners Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 51 +++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 16274a39..9d9bf321 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -23,7 +23,12 @@ jobs: name: Test (Python 3.10) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -57,7 +62,12 @@ jobs: name: Lint (Python 3.10) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -94,7 +104,12 @@ jobs: matrix: python-version: [3.8, 3.9] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -127,14 +142,25 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + + - name: Cleanup runner + run: | + pip cache purge + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" - CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" - CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" + CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64" with: output-dir: wheelhouse @@ -144,7 +170,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-south-1 run: | - which aws || pip install --no-cache-dir awscli --upgrade + which aws || pip install awscli --upgrade aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ --acl public-read \ --delete \ @@ -156,6 +182,17 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + + - name: Cleanup runner + run: | + pip cache purge + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + - name: Set up Environment run: | brew install openblas @@ -165,7 +202,7 @@ jobs: uses: pypa/cibuildwheel@v2.23.3 env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" - CIBW_BEFORE_BUILD: "pip install --no-cache-dir torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install --no-cache-dir -r requirements.txt" + CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" CIBW_ARCHS: x86_64 CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" From 55ead007915993053bc06b8d2cce655dd53371d7 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 17:03:50 +0200 Subject: [PATCH 11/28] clean some jobs not necessary Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 120 ---------------------------- 1 file changed, 120 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 9d9bf321..1a712fcd 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -2,8 +2,6 @@ name: release-build on: workflow_dispatch: - # create: - # tags: 'v**' permissions: id-token: write @@ -16,123 +14,6 @@ env: COS_ENDPOINT: ${{ secrets.COS_ENDPOINT }} jobs: - # ──────────────────────────── - # Test on Python 3.10 - # ──────────────────────────── - test-py310: - name: Test (Python 3.10) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: 3.10.17 - - - name: Install system dependencies - run: | - sudo apt-get -qq update - sudo apt-get install -y ca-certificates libopenblas-dev gcc-9 g++-9 - sudo update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-9 - - - name: Build and install aihwkit wheel - run: | - pip install -r requirements.txt - make build_inplace - - - name: Run pytest - run: | - pip install -r requirements-dev.txt - make pytest - env: - TEST_DATASET: true - - # ──────────────────────────── - # Lint on Python 3.10 - # ──────────────────────────── - lint-py310: - name: Lint (Python 3.10) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: 3.10.17 - - - name: Install system dependencies - run: | - sudo apt-get -qq update - sudo apt-get install -y ca-certificates libopenblas-dev gcc-9 g++-9 - sudo update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-9 - - - name: Build and install aihwkit wheel - run: | - pip install -r requirements.txt - make build_inplace - - - name: Run lint checks - run: | - pip install -r requirements-dev.txt - pip install -r requirements-examples.txt - make pycodestyle - make pylint - make mypy - # ──────────────────────────────────────── - # Stage: Test multiple Python versions - # ──────────────────────────────────────── - test-matrix: - name: Tests (3.8 & 3.9) - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9] - steps: - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system deps - run: | - sudo apt-get -qq update - sudo apt-get install -y ca-certificates libopenblas-dev gcc-9 g++-9 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-9 - - - name: Build and install aihwkit wheel - run: | - pip install -r requirements.txt - make build_inplace - - - name: Run pytest - run: | - pip install -r requirements-dev.txt - make pytest - # ──────────────────────────── # Stage: Build & Deploy Wheels # ──────────────────────────── @@ -190,7 +71,6 @@ jobs: - name: Cleanup runner run: | pip cache purge - sudo apt-get clean sudo rm -rf /var/lib/apt/lists/* - name: Set up Environment From 14d5460994b54184d8be2ae5a15cc4a9d4632e63 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 17:27:22 +0200 Subject: [PATCH 12/28] add compatible macosx version Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 1a712fcd..0766ca34 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -86,6 +86,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" CIBW_ARCHS: x86_64 CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + MACOSX_DEPLOYMENT_TARGET: "10.13" with: output-dir: wheelhouse From 075cd2371608a09db08092cfdc8d827cde7ab5ad Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Tue, 20 May 2025 17:47:58 +0200 Subject: [PATCH 13/28] disk optimization Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 65 +++++++++-------------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 0766ca34..2fdf235c 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -41,7 +41,7 @@ jobs: CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" - CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64" + CIBW_BUILD: "cp310-manylinux_x86_64" with: output-dir: wheelhouse @@ -57,9 +57,18 @@ jobs: --delete \ --endpoint-url https://${{ env.COS_ENDPOINT }} + - name: Cleanup runner + if: always() + run: | + pip cache purge + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* /var/cache/apt/* + rm -rf wheelhouse build dist *.egg-info + build-and-deploy-macos: name: Build & Deploy wheels (macOS) runs-on: macos-latest + needs: build-and-deploy-linux steps: - uses: actions/checkout@v4 @@ -68,11 +77,6 @@ jobs: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - name: Cleanup runner - run: | - pip cache purge - sudo rm -rf /var/lib/apt/lists/* - - name: Set up Environment run: | brew install openblas @@ -83,9 +87,8 @@ jobs: env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" - CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" CIBW_ARCHS: x86_64 - CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" + CIBW_BUILD: "cp310-macosx_x86_64" MACOSX_DEPLOYMENT_TARGET: "10.13" with: output-dir: wheelhouse @@ -101,41 +104,11 @@ jobs: --acl public-read \ --delete \ --endpoint-url https://${{ env.COS_ENDPOINT }} - - # build-and-deploy-windows: - # name: Build & Deploy wheels (Windows) - # runs-on: windows-latest - # env: - # CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.4.1'" - # CIBW_BEFORE_BUILD: "pip install torch==2.4.1 && pip install -r requirements.txt" - # CIBW_BUILD: "cp38-win_amd64 cp39-win_amd64 cp310-win_amd64" - # OPENBLAS_ROOT: C:\\BLAS - # OPENBLAS_ROOT_DIR: C:\\BLAS - # steps: - # - uses: actions/checkout@v3 - - # - name: Install Python & BLAS - # run: | - # choco install python --version=3.8.6 -y - # SET PATH=C:\Python38;C:\Python38\Scripts;%PATH% - # mkdir C:\BLAS - # Invoke-WebRequest -Uri https://github.com/xianyi/OpenBLAS/releases/download/v0.3.12/OpenBLAS-0.3.12-x64.zip -OutFile openblas.zip - # Expand-Archive openblas.zip -DestinationPath C:\BLAS - - # - name: Install cibuildwheel - # run: python -m pip install cibuildwheel==2.8.1 - - # - name: Build wheels - # run: python -m cibuildwheel --output-dir wheelhouse - - # - name: Sync wheels to IBM COS - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} - # AWS_DEFAULT_REGION: us-south-1 - # run: | - # which aws || pip install awscli --upgrade - # aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - # --acl public-read \ - # --delete \ - # --endpoint-url https://${{ env.COS_ENDPOINT }} + + - name: Cleanup runner + if: always() + run: | + pip cache purge + brew cleanup --prune=all + rm -rf ~/Library/Caches/Homebrew/* + rm -rf wheelhouse build dist *.egg-info From 2933d287728d0712251d97bdd69f5b2830be6f8c Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Wed, 21 May 2025 13:38:57 +0200 Subject: [PATCH 14/28] update pyproject.toml and modify release-build Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 9 +--- pyproject.toml | 71 ++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 2fdf235c..2377703c 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -53,8 +53,6 @@ jobs: run: | which aws || pip install awscli --upgrade aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - --acl public-read \ - --delete \ --endpoint-url https://${{ env.COS_ENDPOINT }} - name: Cleanup runner @@ -68,7 +66,6 @@ jobs: build-and-deploy-macos: name: Build & Deploy wheels (macOS) runs-on: macos-latest - needs: build-and-deploy-linux steps: - uses: actions/checkout@v4 @@ -87,8 +84,8 @@ jobs: env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" - CIBW_ARCHS: x86_64 - CIBW_BUILD: "cp310-macosx_x86_64" + CIBW_ARCHS: "arm64" + CIBW_BUILD: "cp310-macosx_arm64" MACOSX_DEPLOYMENT_TARGET: "10.13" with: output-dir: wheelhouse @@ -101,8 +98,6 @@ jobs: run: | which aws || pip install --no-cache-dir awscli --upgrade aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - --acl public-read \ - --delete \ --endpoint-url https://${{ env.COS_ENDPOINT }} - name: Cleanup runner diff --git a/pyproject.toml b/pyproject.toml index 26184e86..32b2ce5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = [ +requires = [" "setuptools>=66.0.0", "wheel", "ninja", @@ -15,5 +15,74 @@ requires = [ "protobuf>=4.21.6", "tqdm", "mypy==0.991" + "ninja; platform_system!='Windows'" ] build-backend = "setuptools.build_meta" +backend-path = ["."] + +[project] +name = "aihwkit" +version = {file = "src/aihwkit/VERSION.txt"} +description = "IBM Analog Hardware Acceleration Kit" +readme = "README.md" +license = {text = "MIT"} +authors = [ + {name = "IBM Research", email = "aihwkit@us.ibm.com"}, +] +requires-python = ">=3.7" +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Environment :: GPU :: NVIDIA CUDA", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Typing :: Typed", +] +keywords = [ + "ai", + "analog", + "rpu", + "torch", + "memristor", + "pcm", + "reram", + "crossbar", + "in-memory", + "nvm", + "non-von-neumann", + "non-volatile memory", + "phase-change material", +] +dependencies = [ + "torch>=1.9", # Note: TORCH_VERSION_SPECIFIER logic needs to be handled during build or by user + "torchvision", + "scipy", + "requests>=2.25,<3", + "numpy>=1.22", + "protobuf>=4.21.6", +] + +[project.urls] +Homepage = "https://github.com/IBM/aihwkit" + +[project.optional-dependencies] +visualization = ["matplotlib>=3.0"] +fitting = ["lmfit"] +bert = ["transformers", "evaluate", "datasets", "wandb", "tensorboard"] + +[tool.setuptools] +package-dir = {"" = "src"} +packages = ["aihwkit"] # Assuming find_packages("src") primarily finds "aihwkit" +include-package-data = true + +[tool.setuptools.package-data] +aihwkit = ["VERSION.txt"] + +[tool.skbuild] +cmake.minimum-version = "3.18" From dbb5389794093d5805bf24f9a71f3ea8e8f33e90 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Thu, 22 May 2025 19:51:17 +0200 Subject: [PATCH 15/28] change versions for build Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 2377703c..8c427d88 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -35,7 +35,7 @@ jobs: sudo rm -rf /var/lib/apt/lists/* - name: Build wheels - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v2.8.1 env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" @@ -78,15 +78,16 @@ jobs: run: | brew install openblas brew link --force openblas + git clone -b aihwkit https://github.com/aihwkit-bot/delocate.git - name: Build wheels - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v2.8.1 env: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" - CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" - CIBW_ARCHS: "arm64" - CIBW_BUILD: "cp310-macosx_arm64" - MACOSX_DEPLOYMENT_TARGET: "10.13" + CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install ./delocate && pip install -r requirements.txt" + CIBW_ARCHS: "x86_64" + CIBW_BUILD: "cp310-macosx_x86_64" + MACOSX_DEPLOYMENT_TARGET: "10.9" with: output-dir: wheelhouse From 29f20d2acd631135ec7e76e8020e83e095882b40 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 14:49:56 +0200 Subject: [PATCH 16/28] move from aws to ibmcloud cos publish Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 59 +++++++++++++++++++---------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 8c427d88..3dbcd3f5 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -3,15 +3,11 @@ name: release-build on: workflow_dispatch: -permissions: - id-token: write - contents: read - env: - COS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} - COS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} - COS_BUCKET: ${{ secrets.COS_BUCKET }} - COS_ENDPOINT: ${{ secrets.COS_ENDPOINT }} + IBM_CLOUD_REGION: ${{ secrets.IBM_CLOUD_REGION }} + IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }} + IBM_CLOUD_RESOURCE_GROUP: ${{ secrets.IBM_CLOUD_RESOURCE_GROUP }} + COS_BUCKET: ${{ secrets.COS_BUCKET }} jobs: # ──────────────────────────── @@ -47,13 +43,24 @@ jobs: - name: Sync wheels to IBM COS env: - AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-south-1 + IBM_CLOUD_REGION: ${{ secrets.IBM_CLOUD_REGION }} + IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }} + IBM_CLOUD_RESOURCE_GROUP: ${{ secrets.IBM_CLOUD_RESOURCE_GROUP }} + COS_BUCKET: ${{ secrets.COS_BUCKET }} run: | - which aws || pip install awscli --upgrade - aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - --endpoint-url https://${{ env.COS_ENDPOINT }} + curl -fsSL https://clis.cloud.ibm.com/install/linux | sh + ibmcloud plugin install cos + ibmcloud login --apikey "$IBMCLOUD_API_KEY" -r "$IBM_CLOUD_REGION" -g "$IBM_CLOUD_RESOURCE_GROUP" + find "wheelhouse" -type f | while read filepath; do + relpath="${filepath#wheelhouse/}" + dest="$relpath" + echo "Uploading $filepath to $dest" + ibmcloud cos upload \ + --bucket "$COS_BUCKET" \ + --key "$dest" \ + --file "$filepath" \ + --region "$IBM_CLOUD_REGION" + done - name: Cleanup runner if: always() @@ -93,14 +100,24 @@ jobs: - name: Sync wheels to IBM COS env: - AWS_ACCESS_KEY_ID: ${{ secrets.COS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.COS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-south-1 + IBM_CLOUD_REGION: ${{ secrets.IBM_CLOUD_REGION }} + IBM_CLOUD_API_KEY: ${{ secrets.IBM_CLOUD_API_KEY }} + IBM_CLOUD_RESOURCE_GROUP: ${{ secrets.IBM_CLOUD_RESOURCE_GROUP }} + COS_BUCKET: ${{ secrets.COS_BUCKET }} run: | - which aws || pip install --no-cache-dir awscli --upgrade - aws s3 sync wheelhouse/ s3://${{ env.COS_BUCKET }}/ \ - --endpoint-url https://${{ env.COS_ENDPOINT }} - + curl -fsSL https://clis.cloud.ibm.com/install/osx | sh + ibmcloud plugin install cos + ibmcloud login --apikey "$IBMCLOUD_API_KEY" -r "$IBM_CLOUD_REGION" -g "$IBM_CLOUD_RESOURCE_GROUP" + find "wheelhouse" -type f | while read filepath; do + relpath="${filepath#wheelhouse/}" + dest="$relpath" + echo "Uploading $filepath to $dest" + ibmcloud cos upload \ + --bucket "$COS_BUCKET" \ + --key "$dest" \ + --file "$filepath" \ + --region "$IBM_CLOUD_REGION" + done - name: Cleanup runner if: always() run: | From 7846641d57e94e2fe3cf4d9c56a4681ed697a7e2 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 14:52:44 +0200 Subject: [PATCH 17/28] fix pyproject.toml syntax error Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 32b2ce5e..3aed711d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = [" +requires = [ "setuptools>=66.0.0", "wheel", "ninja", From db4d5844e56b69e613c147ee659a434a39d0f5a6 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 14:59:31 +0200 Subject: [PATCH 18/28] fix pyproject.toml syntax error Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3aed711d..ca472add 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,8 +14,8 @@ requires = [ "numpy>=1.22,<2", "protobuf>=4.21.6", "tqdm", - "mypy==0.991" - "ninja; platform_system!='Windows'" + "mypy==0.991", + "ninja; platform_system!='Windows'", ] build-backend = "setuptools.build_meta" backend-path = ["."] From 4bbbc2f277171fe56f15b2f7f7961ca71df7e10a Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 15:10:27 +0200 Subject: [PATCH 19/28] remove backend path Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ca472add..f828a7f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ requires = [ "ninja; platform_system!='Windows'", ] build-backend = "setuptools.build_meta" -backend-path = ["."] [project] name = "aihwkit" From 0dd5a58dd9eb97d0ee47b6a1dd27ff7010e65a2f Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 15:14:04 +0200 Subject: [PATCH 20/28] fix pyproject.toml syntax error Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- pyproject.toml | 67 -------------------------------------------------- 1 file changed, 67 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f828a7f1..6dc61748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,70 +18,3 @@ requires = [ "ninja; platform_system!='Windows'", ] build-backend = "setuptools.build_meta" - -[project] -name = "aihwkit" -version = {file = "src/aihwkit/VERSION.txt"} -description = "IBM Analog Hardware Acceleration Kit" -readme = "README.md" -license = {text = "MIT"} -authors = [ - {name = "IBM Research", email = "aihwkit@us.ibm.com"}, -] -requires-python = ">=3.7" -classifiers = [ - "Development Status :: 4 - Beta", - "Environment :: Console", - "Environment :: GPU :: NVIDIA CUDA", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Typing :: Typed", -] -keywords = [ - "ai", - "analog", - "rpu", - "torch", - "memristor", - "pcm", - "reram", - "crossbar", - "in-memory", - "nvm", - "non-von-neumann", - "non-volatile memory", - "phase-change material", -] -dependencies = [ - "torch>=1.9", # Note: TORCH_VERSION_SPECIFIER logic needs to be handled during build or by user - "torchvision", - "scipy", - "requests>=2.25,<3", - "numpy>=1.22", - "protobuf>=4.21.6", -] - -[project.urls] -Homepage = "https://github.com/IBM/aihwkit" - -[project.optional-dependencies] -visualization = ["matplotlib>=3.0"] -fitting = ["lmfit"] -bert = ["transformers", "evaluate", "datasets", "wandb", "tensorboard"] - -[tool.setuptools] -package-dir = {"" = "src"} -packages = ["aihwkit"] # Assuming find_packages("src") primarily finds "aihwkit" -include-package-data = true - -[tool.setuptools.package-data] -aihwkit = ["VERSION.txt"] - -[tool.skbuild] -cmake.minimum-version = "3.18" From 3125e45c60abdf1c80792f42743fe232d1ebaeb7 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 15:20:04 +0200 Subject: [PATCH 21/28] bump macosx target version from 10.9 to 10.13 Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 3dbcd3f5..3af54f26 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -94,7 +94,7 @@ jobs: CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install ./delocate && pip install -r requirements.txt" CIBW_ARCHS: "x86_64" CIBW_BUILD: "cp310-macosx_x86_64" - MACOSX_DEPLOYMENT_TARGET: "10.9" + MACOSX_DEPLOYMENT_TARGET: "10.13" with: output-dir: wheelhouse From c71c8fd9a2a8d14a25662ff9672d7b79f6c4daf6 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 15:32:11 +0200 Subject: [PATCH 22/28] fix pyproject.toml typo errpr Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 3af54f26..5a74655e 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -50,7 +50,7 @@ jobs: run: | curl -fsSL https://clis.cloud.ibm.com/install/linux | sh ibmcloud plugin install cos - ibmcloud login --apikey "$IBMCLOUD_API_KEY" -r "$IBM_CLOUD_REGION" -g "$IBM_CLOUD_RESOURCE_GROUP" + ibmcloud login --apikey "$IBM_CLOUD_API_KEY" -r "$IBM_CLOUD_REGION" -g "$IBM_CLOUD_RESOURCE_GROUP" find "wheelhouse" -type f | while read filepath; do relpath="${filepath#wheelhouse/}" dest="$relpath" @@ -107,7 +107,7 @@ jobs: run: | curl -fsSL https://clis.cloud.ibm.com/install/osx | sh ibmcloud plugin install cos - ibmcloud login --apikey "$IBMCLOUD_API_KEY" -r "$IBM_CLOUD_REGION" -g "$IBM_CLOUD_RESOURCE_GROUP" + ibmcloud login --apikey "$IBM_CLOUD_API_KEY" -r "$IBM_CLOUD_REGION" -g "$IBM_CLOUD_RESOURCE_GROUP" find "wheelhouse" -type f | while read filepath; do relpath="${filepath#wheelhouse/}" dest="$relpath" From b76da3f37915f98999efb78f94a98dda874728c4 Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 15:43:54 +0200 Subject: [PATCH 23/28] add multiple python version Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 5a74655e..5abddb79 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -37,7 +37,7 @@ jobs: CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" - CIBW_BUILD: "cp310-manylinux_x86_64" + CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" with: output-dir: wheelhouse @@ -93,7 +93,7 @@ jobs: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install ./delocate && pip install -r requirements.txt" CIBW_ARCHS: "x86_64" - CIBW_BUILD: "cp310-macosx_x86_64" + CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-macosx_x86_64" MACOSX_DEPLOYMENT_TARGET: "10.13" with: output-dir: wheelhouse From 36971429f5affca01f8a8dd56e94433ed16dcefb Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 15:53:56 +0200 Subject: [PATCH 24/28] fix naming version for macosx builds Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 5abddb79..3759370d 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -93,7 +93,7 @@ jobs: CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install ./delocate && pip install -r requirements.txt" CIBW_ARCHS: "x86_64" - CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-macosx_x86_64" + CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" MACOSX_DEPLOYMENT_TARGET: "10.13" with: output-dir: wheelhouse From 11b76a7fe1fd1d14cba26ef713253a9c23cbccfd Mon Sep 17 00:00:00 2001 From: Pablo Carmona Gonzalez Date: Fri, 23 May 2025 19:29:55 +0200 Subject: [PATCH 25/28] add cicd status badge to README Signed-off-by: Pablo Carmona Gonzalez Signed-off-by: PabloCarmona --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d22515bc..69ebd963 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # IBM Analog Hardware Acceleration Kit ![PyPI](https://img.shields.io/pypi/v/aihwkit) +[![test-and-lint](https://github.com/IBM/aihwkit/actions/workflows/test-and-lint.yml/badge.svg)](https://github.com/IBM/aihwkit/actions/workflows/test-and-lint.yml) +[![release-build](https://github.com/IBM/aihwkit/actions/workflows/release-build.yml/badge.svg)](https://github.com/IBM/aihwkit/actions/workflows/release-build.yml) [![Documentation Status](https://readthedocs.org/projects/aihwkit/badge/?version=latest)](https://aihwkit.readthedocs.io/en/latest/?badge=latest) ![PyPI - License](https://img.shields.io/pypi/l/aihwkit) [![arXiv](https://img.shields.io/badge/arXiv-2104.02184-green.svg)](https://arxiv.org/abs/2104.02184) From beb9cd0045206e6a3e05e6d74798c54c693c7948 Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Wed, 28 May 2025 17:28:41 +0200 Subject: [PATCH 26/28] fix problem with Pytorch versions Signed-off-by: Pablo Gonzalez Signed-off-by: PabloCarmona --- setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index e1098fa3..01f38568 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ from skbuild import setup INSTALL_REQUIRES = [ - "torch{}".format(os.getenv("TORCH_VERSION_SPECIFIER", ">=1.9")), + "torch{}".format(os.getenv("TORCH_VERSION_SPECIFIER", ">=1.9,<2.7")), "torchvision", "scipy", "requests>=2.25,<3", @@ -20,13 +20,12 @@ "protobuf>=4.21.6", ] - def get_version() -> str: - """Get the package version.""" + """Get the package version for CUDA enabled wheels.""" version_path = os.path.join(os.path.dirname(__file__), "src", "aihwkit", "VERSION.txt") with open(version_path, encoding="utf-8") as version_file: - return version_file.read().strip() - + v = version_file.read().strip() + return f"{v}{os.getenv('AIHWKIT_VERSION_SUFFIX', '')}" def get_long_description() -> str: """Get the package long description.""" From f77f7ef3e9500039f2206e5438133f6a2e6582f1 Mon Sep 17 00:00:00 2001 From: PabloCarmona Date: Wed, 28 May 2025 17:35:44 +0200 Subject: [PATCH 27/28] add fix to release-build workflow Signed-off-by: PabloCarmona --- .github/workflows/release-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 3759370d..cd7dea34 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -33,8 +33,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.8.1 env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" - CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1' MAKEFLAGS='-j4'" + CIBW_BEFORE_BUILD: "pip install torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install -r requirements.txt" CIBW_MANYLINUX_X86_64_IMAGE: "aihwkit/manylinux2014_x86_64_aihwkit" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair -w {dest_dir} {wheel} --exclude libtorch_python.so" CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64" @@ -90,8 +90,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.8.1 env: - CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='>=2.1'" - CIBW_BEFORE_BUILD: "pip install torch>=2.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install ./delocate && pip install -r requirements.txt" + CIBW_ENVIRONMENT: "TORCH_VERSION_SPECIFIER='==2.3.1'" + CIBW_BEFORE_BUILD: "pip install torch==2.3.1 torchvision -f https://download.pytorch.org/whl/torch_stable.html && pip install ./delocate && pip install -r requirements.txt" CIBW_ARCHS: "x86_64" CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" MACOSX_DEPLOYMENT_TARGET: "10.13" From a3247662b1d58007cdeb641ba0c32134a32369e9 Mon Sep 17 00:00:00 2001 From: PabloCarmona Date: Wed, 28 May 2025 18:01:10 +0200 Subject: [PATCH 28/28] fix lint for naming Signed-off-by: PabloCarmona --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 01f38568..d1e01d85 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,8 @@ def get_version() -> str: """Get the package version for CUDA enabled wheels.""" version_path = os.path.join(os.path.dirname(__file__), "src", "aihwkit", "VERSION.txt") with open(version_path, encoding="utf-8") as version_file: - v = version_file.read().strip() - return f"{v}{os.getenv('AIHWKIT_VERSION_SUFFIX', '')}" + version_number = version_file.read().strip() + return f"{version_number}{os.getenv('AIHWKIT_VERSION_SUFFIX', '')}" def get_long_description() -> str: """Get the package long description."""