CUDA out of memory

[luyr]

When trying to run training, I face the following CUDA out of memory issue. The program tried to allocate 152638.31 GiB memory, which is quite weired.

Traceback (most recent call last):
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/scripts/train.py", line 87, in main
    trainer.fit(model_module, datamodule=data_module, ckpt_path=ckpt_path)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 545, in fit
    call._call_and_handle_interrupt(
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 43, in _call_and_handle_interrupt
    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 102, in launch
    return function(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 581, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 990, in _run
    results = self._run_stage()
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 1036, in _run_stage
    self.fit_loop.run()
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 202, in run
    self.advance()
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 359, in advance
    self.epoch_loop.run(self._data_fetcher)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 136, in run
    self.advance(data_fetcher)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 240, in advance
    batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 187, in run
    self._optimizer_step(batch_idx, closure)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 265, in _optimizer_step
    call._call_lightning_module_hook(
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 157, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/core/module.py", line 1282, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/core/optimizer.py", line 151, in step
    step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/strategies/ddp.py", line 263, in optimizer_step
    optimizer_output = super().optimizer_step(optimizer, closure, model, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/strategies/strategy.py", line 230, in optimizer_step
    return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 117, in optimizer_step
    return optimizer.step(closure=closure, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/optim/lr_scheduler.py", line 68, in wrapper
    return wrapped(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/optim/optimizer.py", line 373, in wrapper
    out = func(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/optim/optimizer.py", line 76, in _use_grad
    ret = func(self, *args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/optim/adamw.py", line 161, in step
    loss = closure()
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 104, in _wrap_closure
    closure_result = closure()
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 126, in closure
    step_output = self._step_fn()
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 315, in _training_step
    training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 309, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/strategies/strategy.py", line 381, in training_step
    return self._forward_redirection(self.model, self.lightning_module, "training_step", *args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/strategies/strategy.py", line 628, in __call__
    wrapper_output = wrapper_module(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward
    else self._run_ddp_forward(*inputs, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
    return self.module(*inputs, **kwargs)  # type: ignore[index]
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/pytorch_lightning/strategies/strategy.py", line 621, in wrapped_forward
    out = method(*_args, **_kwargs)
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/model_module.py", line 55, in training_step
    return self.shared_step(batch, 'train', on_step = True,
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/model_module.py", line 27, in shared_step
    pred = self(batch)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/model_module.py", line 24, in forward
    return self.backbone(batch)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/GaussianLSS.py", line 104, in forward
    x, num_gaussians = self.gs_render(features, means3D, cov3D, opacities)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/GaussianLSS.py", line 175, in forward
    rendered_bev, _ = self.rasterizer(
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py", line 211, in forward
    return rasterize_gaussians(
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py", line 32, in rasterize_gaussians
    return _RasterizeGaussians.apply(
  File "/home/luy1syv/.conda/envs/GaussianLSS/lib/python3.9/site-packages/torch/autograd/function.py", line 539, in apply
    return super().apply(*args, **kwargs)  # type: ignore[misc]
  File "/home/luy1syv/yiren_workspace/GaussianLSS-main/GaussianLSS/model/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py", line 92, in forward
    num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args)
torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 152638.31 GiB. GPU 0 has a total capacty of 79.25 GiB of which 67.39 GiB is free. Including non-PyTorch memory, this process has 11.85 GiB memory in use. Of the allocated memory 11.03 GiB is allocated by PyTorch, and 220.12 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF

[wjzhang642]

I encouter the same issue.

[luyr]

I encouter the same issue.
This issue may be caused by a version mismatch between the installed cudatoolkit and the CUDA version used to build PyTorch. I resolved it by creating a new environment and reinstalling both PyTorch and the rasterizer.

[wjzhang642]

I encouter the same issue.
This issue may be caused by a version mismatch between the installed cudatoolkit and the CUDA version used to build PyTorch. I resolved it by creating a new environment and reinstalling both PyTorch and the rasterizer.

Thanks for your reply! Could you kindly share a install script or the environment packages list?
My torch version is 2.1.0+cu121, and nvcc version is also 12.1:

>>>pip list
Package                     Version           
--------------------------- ------------------ 
aiohappyeyeballs            2.4.4
aiohttp                     3.10.11
aiosignal                   1.3.1
antlr4-python3-runtime      4.9.3
anyio                       4.5.2
appdirs                     1.4.4
argon2-cffi                 25.1.0
argon2-cffi-bindings        21.2.0
arrow                       1.3.0
asttokens                   3.0.0
async-lru                   2.0.4
async-timeout               5.0.1
attrs                       25.3.0
babel                       2.17.0
backcall                    0.2.0
beautifulsoup4              4.13.4
bleach                      6.1.0
cachetools                  5.5.2
certifi                     2022.12.7
cffi                        1.17.1
charset-normalizer          2.1.1
click                       8.1.8
comm                        0.2.3
cycler                      0.12.1
debugpy                     1.8.16
decorator                   5.2.1
defusedxml                  0.7.1
descartes                   1.1.0
diff-gaussian-rasterization 0.0.0           
docker-pycreds              0.4.0
efficientnet_pytorch        0.7.1
einops                      0.3.2
exceptiongroup              1.3.0
executing                   2.2.0
fastjsonschema              2.21.2
filelock                    3.13.1
fire                        0.7.1
fonttools                   4.57.0
fqdn                        1.5.1
frozenlist                  1.5.0
fsspec                      2024.6.1
fvcore                      0.1.5.post20210825
gitdb                       4.0.12
GitPython                   3.1.45
h11                         0.16.0
hf-xet                      1.1.7
httpcore                    1.0.9
httpx                       0.28.1
huggingface-hub             0.34.4
hydra-core                  1.3.2
idna                        3.4
imageio                     2.35.1
imgaug                      0.4.0
importlib_metadata          8.5.0
importlib_resources         6.4.5
iopath                      0.1.10
ipykernel                   6.29.5
ipython                     8.12.3
ipywidgets                  8.1.7
isoduration                 20.11.0
jedi                        0.19.2
Jinja2                      3.1.4
joblib                      1.4.2
json5                       0.12.1
jsonpointer                 3.0.0
jsonschema                  4.23.0
jsonschema-specifications   2023.12.1
jupyter                     1.1.1
jupyter_client              8.6.3
jupyter-console             6.6.3
jupyter_core                5.8.1
jupyter-events              0.10.0
jupyter-lsp                 2.2.6
jupyter_server              2.14.2
jupyter_server_terminals    0.5.3
jupyterlab                  4.3.8
jupyterlab_pygments         0.3.0
jupyterlab_server           2.27.3
jupyterlab_widgets          3.0.15
kiwisolver                  1.4.7
lazy_loader                 0.4
lightning-utilities         0.11.9
MarkupSafe                  2.1.5
matplotlib                  3.5.3
matplotlib-inline           0.1.7
mistune                     3.1.3
mpmath                      1.3.0
multidict                   6.1.0
nbclient                    0.10.1
nbconvert                   7.16.6
nbformat                    5.10.4
nest-asyncio                1.6.0
networkx                    3.0
notebook                    7.3.3
notebook_shim               0.2.4
numpy                       1.24.1
nuscenes-devkit             1.1.11
omegaconf                   2.3.0
opencv-python               4.12.0.88
overrides                   7.7.0
packaging                   25.0
pandocfilters               1.5.1
parso                       0.8.4
pathtools                   0.1.2
pexpect                     4.9.0
pickleshare                 0.7.5
pillow                      10.2.0
pip                         24.2
pkgutil_resolve_name        1.3.10
platformdirs                4.3.6
portalocker                 3.0.0
prometheus_client           0.21.1
prompt_toolkit              3.0.51
propcache                   0.2.0
protobuf                    4.25.8
psutil                      7.0.0
ptyprocess                  0.7.0
pure_eval                   0.2.3
pycocotools                 2.0.7
pycparser                   2.22
Pygments                    2.19.2
pyparsing                   3.1.4
pyquaternion                0.9.9
python-dateutil             2.9.0.post0
python-json-logger          3.3.0
pytorch-lightning           2.1.0
pytz                        2025.2
PyWavelets                  1.4.1
PyYAML                      6.0.2
pyzmq                       27.0.1
referencing                 0.35.1
requests                    2.32.4
rfc3339-validator           0.1.4
rfc3986-validator           0.1.1
rpds-py                     0.20.1
safetensors                 0.5.3
scikit-image                0.21.0
scikit-learn                1.3.2
scipy                       1.10.1
Send2Trash                  1.8.3
sentry-sdk                  2.35.0
setproctitle                1.3.6
setuptools                  60.2.0
Shapely                     1.8.5.post1
six                         1.17.0
smmap                       5.0.2
sniffio                     1.3.1
soupsieve                   2.7
stack-data                  0.6.3
sympy                       1.13.3
tabulate                    0.9.0
termcolor                   2.4.0
terminado                   0.18.1
threadpoolctl               3.5.0
tifffile                    2023.7.10
timm                        0.9.16
tinycss2                    1.2.1
tomli                       2.2.1
torch                       2.1.0+cu121
torchaudio                  2.1.0+cu121
torchmetrics                1.5.2
torchvision                 0.16.0+cu121
tornado                     6.4.2
tqdm                        4.65.2
traitlets                   5.14.3
triton                      2.1.0
types-python-dateutil       2.9.0.20241206
typing_extensions           4.12.2
uri-template                1.3.0
urllib3                     1.26.13
wandb                       0.15.5
wcwidth                     0.2.13
webcolors                   24.8.0
webencodings                0.5.1
websocket-client            1.8.0
wheel                       0.44.0
widgetsnbextension          4.0.14
yacs                        0.1.8
yarl                        1.15.2
zipp                        3.20.2

>>> nvcc -V
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2023 NVIDIA Corporation
Built on Tue_Feb__7_19:32:13_PST_2023
Cuda compilation tools, release 12.1, V12.1.66
Build cuda_12.1.r12.1/compiler.32415258_0