Support various block sizes & Change default block size to 16 (#38)

2023-04-15 09:03:24 -07:00
parent 84eee24e20
commit 0f4b32199e
7 changed files with 594 additions and 611 deletions
--- a/benchmark/benchmark_text_completion.py
+++ b/benchmark/benchmark_text_completion.py
@@ -268,6 +268,7 @@ if __name__ == '__main__':
            f'{model_name}-tp{args.tensor_parallel_size}',
            sample_dir,
            'cacheflow',
+            f'block{args.block_size}',
            f'req-rate-{args.request_rate}',
            f'seed{args.seed}',
            f'duration-{args.duration}',