[Public release 26/04] Introducing Mega MoE, FP4 Indexer and other features/fixes (#304)

* Merge with private repo

* Update README

* Update README

* Update README

* Add PyTorch requirements

* Fix sync scopes for MQA logits (#256)

* Update README

tests/test_lazy_init.py

@@ -1,3 +1,4 @@
+import argparse
 import torch
 import torch.multiprocessing as mp
 import deep_gemm
@@ -8,7 +9,11 @@ def main(local_rank: int):
 
 
 if __name__ == '__main__':
-    procs = [mp.Process(target=main, args=(i, ), ) for i in range(8)]
+    parser = argparse.ArgumentParser(description='Test lazy initialization')
+    parser.add_argument('--num-processes', type=int, default=8, help='Number of processes to spawn (default: 8)')
+    args = parser.parse_args()
+
+    procs = [mp.Process(target=main, args=(i, ), ) for i in range(args.num_processes)]
     for p in procs:
         p.start()
     for p in procs: