[FIX] Fix styles in automatic prefix caching & add a automatic prefix caching benchmark (#3158)

This commit is contained in:
Zhuohan Li
2024-03-03 14:37:18 -08:00
committed by GitHub
parent d65fac2738
commit 996d095c54
4 changed files with 69 additions and 18 deletions

View File

@@ -303,7 +303,10 @@ if __name__ == "__main__":
default="cuda",
choices=["cuda"],
help='device type for vLLM execution, supporting CUDA only currently.')
parser.add_argument("--enable_prefix_caching", action='store_true')
parser.add_argument(
"--enable-prefix-caching",
action='store_true',
help="enable automatic prefix caching for vLLM backend.")
args = parser.parse_args()
if args.tokenizer is None:
args.tokenizer = args.model