biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[sleep mode] clear pytorch cache after sleep (#15248)

Browse Source 
Signed-off-by: <villard@us.ibm.com>
This commit is contained in:
Lionel Villard
2025-04-01 01:58:58 -04:00
committed by GitHub
parent 7e4e709b43
commit 4a9ce1784c
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/device_allocator/cumem.py
View File

@@ -8,6 +8,7 @@
# not sure why, they are created from a different context. # not sure why, they are created from a different context.
# the only successful approach is to call cuda driver API in C. # the only successful approach is to call cuda driver API in C.
import dataclasses import dataclasses
import gc
import os import os
from contextlib import contextmanager from contextlib import contextmanager
from typing import Any, Callable, Dict, Optional, Tuple, Union from typing import Any, Callable, Dict, Optional, Tuple, Union
@@ -204,6 +205,9 @@ class CuMemAllocator:
data.cpu_backup_tensor = cpu_backup_tensor data.cpu_backup_tensor = cpu_backup_tensor
unmap_and_release(handle) unmap_and_release(handle)
gc.collect()
torch.cuda.empty_cache()
def wake_up(self): def wake_up(self):
""" """
Wake up the allocator from sleep mode. Wake up the allocator from sleep mode.