[Bugfix] Restore support for larger block sizes (#11259)
Signed-off-by: Konrad Zawora <kzawora@habana.ai>
This commit is contained in:
@@ -917,6 +917,10 @@ class CacheConfig:
|
||||
raise ValueError(
|
||||
"GPU memory utilization must be less than 1.0. Got "
|
||||
f"{self.gpu_memory_utilization}.")
|
||||
if (current_platform.is_cuda() and self.block_size is not None
|
||||
and self.block_size > 32):
|
||||
raise ValueError("CUDA Paged Attention kernel only supports "
|
||||
f"block sizes up to 32. Got {self.block_size}.")
|
||||
|
||||
def _verify_cache_dtype(self) -> None:
|
||||
if self.cache_dtype == "auto":
|
||||
|
||||
Reference in New Issue
Block a user