OOM issue in Colab with L4

[chandrabhuma]

from transformers import AutoProcessor, AutoModelForImageTextToText
from huggingface_hub import hf_hub_download

MODEL_PATH = "facebook/Perception-LM-1B"
processor = AutoProcessor.from_pretrained(MODEL_PATH, use_fast=True)
model = AutoModelForImageTextToText.from_pretrained(MODEL_PATH).to("cuda")
test_image_file = hf_hub_download(
repo_id="shumingh/perception_lm_test_images",
filename="14496_0.PNG",
repo_type="dataset",
)
conversation = [
{
"role": "user",
"content": [
{
"type": "image",
"url": test_image_file,
},
{"type": "text", "text": "Describe the bar plot in the image."},
],
}
]

inputs = processor.apply_chat_template(
[conversation],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
)
inputs = inputs.to(model.device)
generate_ids = model.generate(**inputs, max_new_tokens=256)
input_length = inputs["input_ids"].shape[1]
generate_ids_without_inputs = generate_ids[:, input_length:]

for output in processor.batch_decode(generate_ids_without_inputs, skip_special_tokens=True):
print(output)
The code gave the following....OutOfMemoryError Traceback (most recent call last)
/tmp/ipython-input-3278415429.py in <cell line: 0>()
31 )
32 inputs = inputs.to(model.device)
---> 33 generate_ids = model.generate(**inputs, max_new_tokens=256)
34 input_length = inputs["input_ids"].shape[1]
35 generate_ids_without_inputs = generate_ids[:, input_length:]

24 frames
/usr/local/lib/python3.12/dist-packages/transformers/integrations/sdpa_attention.py in sdpa_attention_forward(module, query, key, value, attention_mask, dropout, scaling, is_causal, **kwargs)
94 attention_mask = torch.logical_not(attention_mask.bool()).to(query.device)
95
---> 96 attn_output = torch.nn.functional.scaled_dot_product_attention(
97 query,
98 key,

OutOfMemoryError: CUDA out of memory. Tried to allocate 10.83 GiB. GPU 0 has a total capacity of 22.16 GiB of which 4.13 GiB is free. Process 23809 has 18.03 GiB memory in use. Of the allocated memory 17.52 GiB is allocated by PyTorch, and 295.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)....Any help ?