From 49478eb9dc0dca839479f1ad7dfb9913fc2f1f0f Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 09:55:19 +0000 Subject: [PATCH 1/3] Enable execution on Google Colab (T4 GPU) - Added `requirements_colab.txt` compatible with Python 3.10, PyTorch 2.4+, and CUDA 12. - Created `colab_start.ipynb` for automated setup, weight download, and inference. - Fixed `CUDA_HOME` environment variable handling in `notebook/inference.py` to support Linux environments. - Removed Windows-specific dependencies for the Colab environment. --- colab_start.ipynb | 181 +++++++++++++++++++++++++++++++++++++++++ notebook/inference.py | 3 +- requirements_colab.txt | 29 +++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 colab_start.ipynb create mode 100644 requirements_colab.txt diff --git a/colab_start.ipynb b/colab_start.ipynb new file mode 100644 index 00000000..ad40db49 --- /dev/null +++ b/colab_start.ipynb @@ -0,0 +1,181 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SAM 3D Objects - Colab Setup\n", + "\n", + "This notebook sets up the environment and runs a demo inference on Google Colab (T4 GPU recommended)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Clone the repository\n", + "import os\n", + "if not os.path.exists(\"sam-3d-objects\"):\n", + " !git clone https://github.com/facebookresearch/sam-3d-objects.git\n", + " %cd sam-3d-objects\n", + "else:\n", + " %cd sam-3d-objects\n", + " !git pull\n", + "\n", + "# Note: If you are running this from a specific PR or fork, make sure to checkout that branch.\n", + "# Example: !git checkout my-pr-branch\n", + "\n", + "# 2. Install dependencies\n", + "print(\"Installing dependencies... This may take a few minutes.\")\n", + "# Optional: Set FIND_LINKS for faster kaolin install if compatible wheel exists\n", + "# %env PIP_FIND_LINKS=https://nvidia-kaolin.s3.us-east-2.amazonaws.com/torch-2.4.0_cu121.html\n", + "!pip install -r requirements_colab.txt\n", + "\n", + "# 3. Install the package in editable mode\n", + "!pip install -e .\n", + "\n", + "# 4. Patch Hydra (required)\n", + "!python patching/hydra" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download Checkpoints\n", + "You need a Hugging Face token to download the model weights. Accept the license at https://huggingface.co/facebook/sam-3d-objects first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import snapshot_download\n", + "import os\n", + "import shutil\n", + "\n", + "# Function to download weights\n", + "def download_weights(token):\n", + " tag = \"hf\"\n", + " download_dir = f\"checkpoints/{tag}-download\"\n", + " target_dir = f\"checkpoints/{tag}\"\n", + " \n", + " if os.path.exists(target_dir) and os.path.exists(os.path.join(target_dir, \"pipeline.yaml\")):\n", + " print(f\"Checkpoints already exist at {target_dir}\")\n", + " return\n", + " \n", + " print(\"Downloading model weights...\")\n", + " try:\n", + " snapshot_download(\n", + " repo_id=\"facebook/sam-3d-objects\",\n", + " repo_type=\"model\",\n", + " local_dir=download_dir,\n", + " max_workers=1,\n", + " token=token\n", + " )\n", + " \n", + " # Move checkpoints to correct location\n", + " source = os.path.join(download_dir, \"checkpoints\")\n", + " if os.path.exists(source):\n", + " if os.path.exists(target_dir):\n", + " shutil.rmtree(target_dir)\n", + " shutil.move(source, target_dir)\n", + " shutil.rmtree(download_dir)\n", + " print(\"Download complete and files moved.\")\n", + " else:\n", + " # Fallback if structure is different\n", + " if os.path.exists(target_dir):\n", + " shutil.rmtree(target_dir)\n", + " shutil.move(download_dir, target_dir)\n", + " print(\"Download complete (fallback structure).\")\n", + " \n", + " except Exception as e:\n", + " print(f\"Error downloading weights: {e}\")\n", + "\n", + "# Input your HF token here\n", + "try:\n", + " from google.colab import userdata\n", + " token = userdata.get('HF_TOKEN')\n", + "except:\n", + " token = None\n", + "\n", + "if not token:\n", + " print(\"Please enter your Hugging Face Token (input hidden):\")\n", + " from getpass import getpass\n", + " token = getpass()\n", + "\n", + "download_weights(token)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Minimal Inference Code\n", + "import sys\n", + "import os\n", + "sys.path.append(\"notebook\")\n", + "from inference import Inference, load_image, load_single_mask\n", + "import torch\n", + "\n", + "if not torch.cuda.is_available():\n", + " print(\"Warning: CUDA is not available. Inference will be slow or fail.\")\n", + "\n", + "tag = \"hf\"\n", + "config_path = f\"checkpoints/{tag}/pipeline.yaml\"\n", + "\n", + "if not os.path.exists(config_path):\n", + " print(f\"Error: Config not found at {config_path}. Did you download weights?\")\n", + "else:\n", + " # Load model\n", + " print(\"Loading model...\")\n", + " inference = Inference(config_path, compile=False)\n", + "\n", + " # Load dummy image/mask (using one from repo)\n", + " image_path = \"notebook/images/shutterstock_stylish_kidsroom_1640806567/image.png\"\n", + " mask_folder = \"notebook/images/shutterstock_stylish_kidsroom_1640806567\"\n", + " \n", + " print(f\"Processing image: {image_path}\")\n", + " if os.path.exists(image_path):\n", + " image = load_image(image_path)\n", + " mask = load_single_mask(mask_folder, index=14)\n", + "\n", + " # Run model\n", + " output = inference(image, mask, seed=42)\n", + "\n", + " # Save output\n", + " output[\"gs\"].save_ply(\"splat.ply\")\n", + " print(\"Success! Output saved to splat.ply\")\n", + " else:\n", + " print(\"Test image not found.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebook/inference.py b/notebook/inference.py index 94e99a7f..f0acd535 100644 --- a/notebook/inference.py +++ b/notebook/inference.py @@ -2,7 +2,8 @@ import os # not ideal to put that here -os.environ["CUDA_HOME"] = os.environ["CONDA_PREFIX"] +if "CUDA_HOME" not in os.environ and "CONDA_PREFIX" in os.environ: + os.environ["CUDA_HOME"] = os.environ["CONDA_PREFIX"] os.environ["LIDRA_SKIP_INIT"] = "true" import sys diff --git a/requirements_colab.txt b/requirements_colab.txt new file mode 100644 index 00000000..7760a2f7 --- /dev/null +++ b/requirements_colab.txt @@ -0,0 +1,29 @@ +numpy +Pillow +opencv-python +matplotlib +seaborn +gradio +omegaconf +hydra-core +timm +h5py +scikit-image +einops-exts +transformers +accelerate +bitsandbytes +gdown +ninja +torch>=2.4.0 +torchvision +torchaudio +kaolin==0.17.0 +gsplat @ git+https://github.com/nerfstudio-project/gsplat.git@2323de5905d5e90e035f792fe65bad0fedd413e7 +pytorch3d @ git+https://github.com/facebookresearch/pytorch3d.git@75ebeeaea0908c5527e7b1e305fbc7681382db47 +xformers +spconv-cu121 +open3d +pandas +scipy +MoGe @ git+https://github.com/microsoft/MoGe.git@a8c37341bc0325ca99b9d57981cc3bb2bd3e255b From 526f23582f0eb7048f3c3680dd74b33374894e0d Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:21:27 +0000 Subject: [PATCH 2/3] Enable execution on Google Colab (T4 GPU) - Added `requirements_colab.txt` compatible with Python 3.10, PyTorch 2.4+, and CUDA 12. - Created `colab_start.ipynb` for automated setup, weight download, and inference. - Fixed `CUDA_HOME` environment variable handling in `notebook/inference.py` to support Linux environments. - Removed Windows-specific dependencies and incompatible `flash_attn` for the Colab environment. From dfb5382250612c6b05d7c7dfa17128a4998245a4 Mon Sep 17 00:00:00 2001 From: RocKamuS <149111224+RocKamuS@users.noreply.github.com> Date: Fri, 9 Jan 2026 16:40:22 +0300 Subject: [PATCH 3/3] Revert "Colab Support and Environment Fixes" --- colab_start.ipynb | 181 ----------------------------------------- notebook/inference.py | 3 +- requirements_colab.txt | 29 ------- 3 files changed, 1 insertion(+), 212 deletions(-) delete mode 100644 colab_start.ipynb delete mode 100644 requirements_colab.txt diff --git a/colab_start.ipynb b/colab_start.ipynb deleted file mode 100644 index ad40db49..00000000 --- a/colab_start.ipynb +++ /dev/null @@ -1,181 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SAM 3D Objects - Colab Setup\n", - "\n", - "This notebook sets up the environment and runs a demo inference on Google Colab (T4 GPU recommended)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# 1. Clone the repository\n", - "import os\n", - "if not os.path.exists(\"sam-3d-objects\"):\n", - " !git clone https://github.com/facebookresearch/sam-3d-objects.git\n", - " %cd sam-3d-objects\n", - "else:\n", - " %cd sam-3d-objects\n", - " !git pull\n", - "\n", - "# Note: If you are running this from a specific PR or fork, make sure to checkout that branch.\n", - "# Example: !git checkout my-pr-branch\n", - "\n", - "# 2. Install dependencies\n", - "print(\"Installing dependencies... This may take a few minutes.\")\n", - "# Optional: Set FIND_LINKS for faster kaolin install if compatible wheel exists\n", - "# %env PIP_FIND_LINKS=https://nvidia-kaolin.s3.us-east-2.amazonaws.com/torch-2.4.0_cu121.html\n", - "!pip install -r requirements_colab.txt\n", - "\n", - "# 3. Install the package in editable mode\n", - "!pip install -e .\n", - "\n", - "# 4. Patch Hydra (required)\n", - "!python patching/hydra" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Download Checkpoints\n", - "You need a Hugging Face token to download the model weights. Accept the license at https://huggingface.co/facebook/sam-3d-objects first." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from huggingface_hub import snapshot_download\n", - "import os\n", - "import shutil\n", - "\n", - "# Function to download weights\n", - "def download_weights(token):\n", - " tag = \"hf\"\n", - " download_dir = f\"checkpoints/{tag}-download\"\n", - " target_dir = f\"checkpoints/{tag}\"\n", - " \n", - " if os.path.exists(target_dir) and os.path.exists(os.path.join(target_dir, \"pipeline.yaml\")):\n", - " print(f\"Checkpoints already exist at {target_dir}\")\n", - " return\n", - " \n", - " print(\"Downloading model weights...\")\n", - " try:\n", - " snapshot_download(\n", - " repo_id=\"facebook/sam-3d-objects\",\n", - " repo_type=\"model\",\n", - " local_dir=download_dir,\n", - " max_workers=1,\n", - " token=token\n", - " )\n", - " \n", - " # Move checkpoints to correct location\n", - " source = os.path.join(download_dir, \"checkpoints\")\n", - " if os.path.exists(source):\n", - " if os.path.exists(target_dir):\n", - " shutil.rmtree(target_dir)\n", - " shutil.move(source, target_dir)\n", - " shutil.rmtree(download_dir)\n", - " print(\"Download complete and files moved.\")\n", - " else:\n", - " # Fallback if structure is different\n", - " if os.path.exists(target_dir):\n", - " shutil.rmtree(target_dir)\n", - " shutil.move(download_dir, target_dir)\n", - " print(\"Download complete (fallback structure).\")\n", - " \n", - " except Exception as e:\n", - " print(f\"Error downloading weights: {e}\")\n", - "\n", - "# Input your HF token here\n", - "try:\n", - " from google.colab import userdata\n", - " token = userdata.get('HF_TOKEN')\n", - "except:\n", - " token = None\n", - "\n", - "if not token:\n", - " print(\"Please enter your Hugging Face Token (input hidden):\")\n", - " from getpass import getpass\n", - " token = getpass()\n", - "\n", - "download_weights(token)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Minimal Inference Code\n", - "import sys\n", - "import os\n", - "sys.path.append(\"notebook\")\n", - "from inference import Inference, load_image, load_single_mask\n", - "import torch\n", - "\n", - "if not torch.cuda.is_available():\n", - " print(\"Warning: CUDA is not available. Inference will be slow or fail.\")\n", - "\n", - "tag = \"hf\"\n", - "config_path = f\"checkpoints/{tag}/pipeline.yaml\"\n", - "\n", - "if not os.path.exists(config_path):\n", - " print(f\"Error: Config not found at {config_path}. Did you download weights?\")\n", - "else:\n", - " # Load model\n", - " print(\"Loading model...\")\n", - " inference = Inference(config_path, compile=False)\n", - "\n", - " # Load dummy image/mask (using one from repo)\n", - " image_path = \"notebook/images/shutterstock_stylish_kidsroom_1640806567/image.png\"\n", - " mask_folder = \"notebook/images/shutterstock_stylish_kidsroom_1640806567\"\n", - " \n", - " print(f\"Processing image: {image_path}\")\n", - " if os.path.exists(image_path):\n", - " image = load_image(image_path)\n", - " mask = load_single_mask(mask_folder, index=14)\n", - "\n", - " # Run model\n", - " output = inference(image, mask, seed=42)\n", - "\n", - " # Save output\n", - " output[\"gs\"].save_ply(\"splat.ply\")\n", - " print(\"Success! Output saved to splat.ply\")\n", - " else:\n", - " print(\"Test image not found.\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebook/inference.py b/notebook/inference.py index f0acd535..94e99a7f 100644 --- a/notebook/inference.py +++ b/notebook/inference.py @@ -2,8 +2,7 @@ import os # not ideal to put that here -if "CUDA_HOME" not in os.environ and "CONDA_PREFIX" in os.environ: - os.environ["CUDA_HOME"] = os.environ["CONDA_PREFIX"] +os.environ["CUDA_HOME"] = os.environ["CONDA_PREFIX"] os.environ["LIDRA_SKIP_INIT"] = "true" import sys diff --git a/requirements_colab.txt b/requirements_colab.txt deleted file mode 100644 index 7760a2f7..00000000 --- a/requirements_colab.txt +++ /dev/null @@ -1,29 +0,0 @@ -numpy -Pillow -opencv-python -matplotlib -seaborn -gradio -omegaconf -hydra-core -timm -h5py -scikit-image -einops-exts -transformers -accelerate -bitsandbytes -gdown -ninja -torch>=2.4.0 -torchvision -torchaudio -kaolin==0.17.0 -gsplat @ git+https://github.com/nerfstudio-project/gsplat.git@2323de5905d5e90e035f792fe65bad0fedd413e7 -pytorch3d @ git+https://github.com/facebookresearch/pytorch3d.git@75ebeeaea0908c5527e7b1e305fbc7681382db47 -xformers -spconv-cu121 -open3d -pandas -scipy -MoGe @ git+https://github.com/microsoft/MoGe.git@a8c37341bc0325ca99b9d57981cc3bb2bd3e255b