[CI/Build] Split out MM pooling tests (#37542)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

.buildkite/test-amd.yaml

@@ -944,48 +944,63 @@ steps:
   - export MIOPEN_DEBUG_CONV_GEMM=0
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt
 
-- label: Multi-Modal Models Test (Extended) 1 # 60min
+- label: Multi-Modal Models Test (Extended Generation 1) # 60min
   timeout_in_minutes: 120
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
   agent_pool: mi250_1
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
+  - tests/models/multimodal/test_mapping.py
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
+    - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
+    - pytest -v -s models/multimodal/test_mapping.py
 
-- label: Multi-Modal Models Test (Extended) 2 #60min
+- label: Multi-Modal Models Test (Extended Generation 2) # 60min
   timeout_in_minutes: 120
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
   agent_pool: mi250_1
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
-- label: Multi-Modal Models Test (Extended) 3 # 75min
+- label: Multi-Modal Models Test (Extended Generation 3) # 75min
   timeout_in_minutes: 150
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
   agent_pool: mi250_1
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
+- label: Multi-Modal Models Test (Extended Pooling) # 60min
+  timeout_in_minutes: 120
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
+  agent_pool: mi250_1
+  optional: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/multimodal/pooling
+  commands:
+    - export MIOPEN_DEBUG_CONV_DIRECT=0
+    - export MIOPEN_DEBUG_CONV_GEMM=0
+    - pytest -v -s models/multimodal/pooling -m 'not core_model'
+
 - label: Quantized Models Test # 45 min
   timeout_in_minutes: 60
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90a]
@@ -2450,7 +2465,7 @@ steps:
   - export MIOPEN_DEBUG_CONV_GEMM=0
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt
 
-- label: Multi-Modal Models Test (Extended) 1 # 60min
+- label: Multi-Modal Models Test (Extended 1) # 60min
   timeout_in_minutes: 120
   mirror_hardwares: [amdexperimental, amdproduction]
   agent_pool: mi325_1
@@ -2458,14 +2473,16 @@ steps:
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
+  - tests/models/multimodal/test_mapping.py
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
+    - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
+    - pytest -v -s models/multimodal/test_mapping.py
 
-- label: Multi-Modal Models Test (Extended) 2 #60min
+- label: Multi-Modal Models Test (Extended 2) # 60min
   timeout_in_minutes: 120
   mirror_hardwares: [amdexperimental, amdproduction]
   agent_pool: mi325_1
@@ -2473,14 +2490,14 @@ steps:
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
-- label: Multi-Modal Models Test (Extended) 3 # 75min
+- label: Multi-Modal Models Test (Extended 3) # 75min
   timeout_in_minutes: 150
   mirror_hardwares: [amdexperimental, amdproduction]
   agent_pool: mi325_1
@@ -2488,13 +2505,27 @@ steps:
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
+- label: Multi-Modal Models Test (Extended Pooling) # 60min
+  timeout_in_minutes: 120
+  mirror_hardwares: [amdexperimental, amdproduction]
+  agent_pool: mi325_1
+  # grade: Blocking
+  optional: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/multimodal/pooling
+  commands:
+    - export MIOPEN_DEBUG_CONV_DIRECT=0
+    - export MIOPEN_DEBUG_CONV_GEMM=0
+    - pytest -v -s models/multimodal/pooling -m 'not core_model'
+
 - label: Quantized Models Test # 45 min
   timeout_in_minutes: 60
   mirror_hardwares: [amdexperimental, amdproduction]
@@ -4175,48 +4206,63 @@ steps:
   - export MIOPEN_DEBUG_CONV_GEMM=0
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt
 
-- label: Multi-Modal Models Test (Extended) 1 # 60min
+- label: Multi-Modal Models Test (Extended 1) # 60min
   timeout_in_minutes: 120
   mirror_hardwares: [amdexperimental]
   agent_pool: mi355_1
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
+  - tests/models/multimodal/test_mapping.py
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
+    - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
+    - pytest -v -s models/multimodal/test_mapping.py
 
-- label: Multi-Modal Models Test (Extended) 2 #60min
+- label: Multi-Modal Models Test (Extended 2) # 60min
   timeout_in_minutes: 120
   mirror_hardwares: [amdexperimental]
   agent_pool: mi355_1
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
-- label: Multi-Modal Models Test (Extended) 3 # 75min
+- label: Multi-Modal Models Test (Extended 3) # 75min
   timeout_in_minutes: 150
   mirror_hardwares: [amdexperimental]
   agent_pool: mi355_1
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - export MIOPEN_DEBUG_CONV_DIRECT=0
     - export MIOPEN_DEBUG_CONV_GEMM=0
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
+- label: Multi-Modal Models Test (Extended Pooling) # 60min
+  timeout_in_minutes: 120
+  mirror_hardwares: [amdexperimental]
+  agent_pool: mi355_1
+  optional: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/multimodal/pooling
+  commands:
+    - export MIOPEN_DEBUG_CONV_DIRECT=0
+    - export MIOPEN_DEBUG_CONV_GEMM=0
+    - pytest -v -s models/multimodal/pooling -m 'not core_model'
+
 - label: Quantized Models Test # 45 min
   timeout_in_minutes: 60
   mirror_hardwares: [amdexperimental, amdproduction]

.buildkite/test_areas/models_multimodal.yaml

@@ -62,7 +62,7 @@ steps:
       depends_on:
       - image-build-amd
 
-- label: Multi-Modal Processor Test (CPU)
+- label: Multi-Modal Processor (CPU)
   depends_on: 
   - image-build-cpu
   timeout_in_minutes: 60
@@ -95,34 +95,44 @@ steps:
   commands:
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
 
-- label: Multi-Modal Models (Extended) 1
+- label: Multi-Modal Models (Extended Generation 1)
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
+  - tests/models/multimodal/test_mapping.py
   commands:
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
+    - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
+    - pytest -v -s models/multimodal/test_mapping.py
   mirror:
     amd:
       device: mi325_1
       depends_on:
       - image-build-amd
 
-- label: Multi-Modal Models (Extended) 2
+- label: Multi-Modal Models (Extended Generation 2)
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
-- label: Multi-Modal Models (Extended) 3
+- label: Multi-Modal Models (Extended Generation 3)
   optional: true
   source_file_dependencies:
   - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
   commands:
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
+
+- label: Multi-Modal Models (Extended Pooling)
+  optional: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/multimodal/pooling
+  commands:
+    - pytest -v -s models/multimodal/pooling -m 'not core_model'