[Bug][CLI] Allow users to disable prefix caching explicitly (#10724)

Signed-off-by: rickyx <rickyx@anyscale.com>
This commit is contained in:
Ricky Xu
2024-11-27 23:59:28 -08:00
committed by GitHub
parent 278be671a3
commit d9b4b3f069
3 changed files with 45 additions and 3 deletions

View File

@@ -416,9 +416,13 @@ class EngineArgs:
'tokens. This is ignored on neuron devices and '
'set to max-model-len')
parser.add_argument('--enable-prefix-caching',
action='store_true',
help='Enables automatic prefix caching.')
parser.add_argument(
"--enable-prefix-caching",
action=argparse.BooleanOptionalAction,
default=EngineArgs.enable_prefix_caching,
help="Enables automatic prefix caching. "
"Use --no-enable-prefix-caching to disable explicitly.",
)
parser.add_argument('--disable-sliding-window',
action='store_true',
help='Disables sliding window, '