From 56777b5c898dcb92dbe6d64fecfccb670f919a0d Mon Sep 17 00:00:00 2001 From: roikoren755 <26850796+roikoren755@users.noreply.github.com> Date: Tue, 24 Mar 2026 02:49:56 +0200 Subject: [PATCH] [Test] E2E Nemotron-3-Super tests (#36803) Signed-off-by: Roi Koren --- .buildkite/test_areas/lm_eval.yaml | 1 + .../configs/Nemotron-3-Super-120B-A12B-BF16.yaml | 11 +++++++++++ .../gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8.yaml | 11 +++++++++++ .../configs/Nemotron-3-Super-120B-A12B-NVFP4.yaml | 11 +++++++++++ tests/evals/gsm8k/configs/models-blackwell.txt | 1 + tests/evals/gsm8k/configs/models-h200.txt | 2 ++ 6 files changed, 37 insertions(+) create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16.yaml create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8.yaml create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4.yaml diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 29f8cb3bc..39029efe9 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -90,6 +90,7 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt + - label: GPQA Eval (GPT-OSS) (H100) timeout_in_minutes: 120 device: h100 diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16.yaml new file mode 100644 index 000000000..d9110efaa --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16.yaml @@ -0,0 +1,11 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 8 + --enable-expert-parallel + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8.yaml new file mode 100644 index 000000000..73a8e9609 --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8.yaml @@ -0,0 +1,11 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 8 + --enable-expert-parallel + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4.yaml new file mode 100644 index 000000000..50f097319 --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4.yaml @@ -0,0 +1,11 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 2 + --enable-expert-parallel + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/models-blackwell.txt b/tests/evals/gsm8k/configs/models-blackwell.txt index c27031d25..2936fa891 100644 --- a/tests/evals/gsm8k/configs/models-blackwell.txt +++ b/tests/evals/gsm8k/configs/models-blackwell.txt @@ -5,3 +5,4 @@ DeepSeek-V2-Lite-Instruct-FP8.yaml Qwen3-30B-A3B-NVFP4.yaml Qwen3-Next-80B-A3B-NVFP4-EP2.yaml Qwen3-Next-FP8-EP2.yaml +Nemotron-3-Super-120B-A12B-NVFP4.yaml diff --git a/tests/evals/gsm8k/configs/models-h200.txt b/tests/evals/gsm8k/configs/models-h200.txt index af305836d..255e93b67 100644 --- a/tests/evals/gsm8k/configs/models-h200.txt +++ b/tests/evals/gsm8k/configs/models-h200.txt @@ -2,3 +2,5 @@ DeepSeek-R1-TP.yaml DeepSeek-R1-DP.yaml DeepSeek-V3.2-TP.yaml DeepSeek-V3.2-DP.yaml +Nemotron-3-Super-120B-A12B-BF16.yaml +Nemotron-3-Super-120B-A12B-FP8.yaml