[Kernel] Expand MoE weight loading + Add Fused Marlin MoE Kernel (#7766)

Co-authored-by: ElizaWszola <eliza@neuralmagic.com>
2024-08-27 18:07:09 -04:00
parent ed6f002d33
commit fc911880cc
16 changed files with 2383 additions and 86 deletions
--- a/tests/quantization/test_compressed_tensors.py
+++ b/tests/quantization/test_compressed_tensors.py
@@ -160,4 +160,4 @@ def test_compressed_tensors_kv_cache(vllm_runner):
    model_path = "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"
    with vllm_runner(model_path, kv_cache_dtype="fp8") as llm:
        output = llm.generate_greedy("Hello world!", max_tokens=20)
-        assert output
+        assert output