Thanks for your great work!The project runs without error at the beginning,but I got an error a few minutes later:
Traceback (most recent call last):
File "/data/ccm/Projects/LOCATE/train.py", line 139, in
masks, logits, loss_proto, loss_con = model(exo, ego, aff_label, (epoch, args.warm_epoch))
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
Traceback (most recent call last):
File "/data/ccm/Projects/LOCATE/train.py", line 139, in
masks, logits, loss_proto, loss_con = model(exo, ego, aff_label, (epoch, args.warm_epoch))
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
return forward_call(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/lightning/fabric/wrappers.py", line 121, in forward
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/lightning/fabric/wrappers.py", line 121, in forward
output = self._forward_module(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
output = self._forward_module(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward
return forward_call(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward
else self._run_ddp_forward(*inputs, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
else self._run_ddp_forward(*inputs, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
return self.module(*inputs, **kwargs) # type: ignore[index]
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self.module(*inputs, **kwargs) # type: ignore[index]
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/data/ccm/Projects/LOCATE/models/locate.py", line 82, in forward
_, exo_key, exo_attn = self.vit_model.get_last_key(exo)
File "/data/ccm/Projects/LOCATE/models/dino/vision_transformer.py", line 284, in get_last_key
return forward_call(*args, **kwargs)
File "/data/ccm/Projects/LOCATE/models/locate.py", line 133, in forward
kmeans.fit_predict(exo_aff_desc.contiguous())
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/fast_pytorch_kmeans/kmeans.py", line 163, in fit_predict
x = blk(x)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
assert X.dtype in [torch.half, torch.float, torch.double], "input must be floating point"
AssertionError: input must be floating point
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/data/ccm/Projects/LOCATE/models/dino/vision_transformer.py", line 122, in forward
y, attn = self.attn(self.norm1(x))
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/data/wsc/anaconda3/envs/LOCATE/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/data/ccm/Projects/LOCATE/models/dino/vision_transformer.py", line 93, in forward
attn = (q @ k.transpose(-2, -1)) * self.scale
torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 22.00 MiB. GPU 2 has a total capacty of 23.70 GiB of which 5.62 MiB is free. Process 2009145 has 21.84 GiB memory in use. Including non-PyTorch memory, this process has 1.84 GiB memory in use. Of the allocated memory 414.50 MiB is allocated by PyTorch, and 27.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation. See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
I don't know what's wrong with this project.