deepseek-v4-quant/__pycache__/fp8_to_nvfp4_streaming.cpython-310.pyc

o
<11><>i#R<00>@s<>
dZd
dl
Z
d
dlZd
dlZd
dlZd
dlZd
dlZd
dlmZ
d
dl	m
Z

d
dlmZ
d
dl
Z
d
dlmZ
d
dlmZ
d
dlmZ
gd	<09>
Ze<03>d
<EFBFBD>dd<0C>eD<00>
<01>
<01>
Ze<03>d
<0A>
Zdedefdd<11>Zde
jde
jde
jfdd<15>Ze
jgd<16>
e
jd<17>Ze
jgd<18>
e
jd<17>a dZ!dZ"de
jde
jfdd<1D>Z#de
jde
jde$e
je
jffd d!<21>Z%d"e
jde
jfd#d$<24>Z&Gd%d&<26>d&<26>Z'd'efd(d)<29>Z(Gd*d+<2B>d+<2B>Z)d,e)fd-d.<2E>Z*d/d0<64>Z+e,d1kr<>e+<2B>
dSdS)2u<32>Streaming FP8 → NVFP4 converter for DeepSeek V4 Pro (sgl-project FP8 repackage).

Path A: pure tensor-level conversion. No model loading via transformers, no
calibration. Reads FP8 safetensors shards, dequantizes per-block FP8 to FP32,
re-quantizes to NVFP4 (E2M1 packed in uint8 with FP8 E4M3 per-block scales and
an FP32 per-tensor global scale), and writes new shards.

Key behaviors:
- Joint global scale_2 across (gate_proj, up_proj) pairs of each expert,
  required for vLLM fused MoE kernels.
- Preserves lm_head, embeddings, MoE router gates, norms, V4 indexer/scoring,
  and mHC residual mixing weights at original precision.
- Streams shard-by-shard. Peak working memory is one tensor pair dequantized
  to FP32 (a few hundred MB at most for the largest weights).
- Resumable per output shard.

NVFP4 format reference:
    value = packed_fp4 * weight_scale * weight_scale_2
    where:
        packed_fp4: E2M1 in {0, ±0.5, ±1, ±1.5, ±2, ±3, ±4, ±6}, 2 per byte
        weight_scale: FP8 E4M3, one per 16-element block
        weight_scale_2: FP32 scalar per tensor, global

Usage:
    python fp8_to_nvfp4_streaming.py \
        --src DeepSeek-V4-Pro-FP8 \
        --dst DeepSeek-V4-Pro-NVFP4-streaming \
        --workers 8

Optional:
    --gpu N           Use CUDA device N for the math (default: 0; -1 for CPU)
    --shard-size-gb 5 Target output shard size
    --dry-run         Print what would be done; don't write
<EFBFBD>N)
<01>defaultdict)
<01>ThreadPoolExecutor)
<01>Path)
<01>	safe_open)
<01>	save_file)
<01>tqdm)z.*lm_head.*z.*embed_tokens.*z.*\.(mlp|ffn)\.gate(\.weight)?$z.*norm.*z.*indexer.*z.*hyper_conn.*z	.*\.mhc.*z.*hc_attn.*z
.*hc_ffn.*z.*hc_head.*z.*scoring.*z
.*attn_sink.*z.*compressor\.ape.*z.*tid2eid.*z	.*\.bias$<24>
|c
cs<00>|]	}
d|
<01>d
<01>V
qdS)z(?:<3A>
)N<>)<02>.0<EFBFBD>
pr
r
<00>@/root/nvidia-meeting/deepseek-v4-quant/fp8_to_nvfp4_streaming.py<70>	<genexpr>Js<02>rz"(.*experts\.\d+)\.(w1|w3)\.weight$<24>name<6D>returnc

Cstt
<01>|<00>
<01>
S<00>
N)<03>bool<6F>PRESERVE_RE<52>match)
rr
r
r
<00>is_preservePs
r<00>
weight_fp8<EFBFBD>	scale_invc	Cs
|<00><00>d
ksJd|j
<01><00><02>
<01>
|j
\}}|
<01><00>dkr!|<00><02>|
<01><02>S|
<01><00>dkrU|
<01><03>|kr8|<00><02>|
<01><02><00>d<04>
S|
<01><03>|krI|<00><02>|
<01><02><00>d<03>
Std|
j
<01>d|j
<01><00><04>
<01>
|
j
\}}||d|}||d|}|
<01><02>j|dd<07>j|dd<07>}|d|<02>d|<03>f}|<00><02>|S)	u<>Dequantize a per-block FP8 E4M3 weight to FP32 using its inverse-scale tensor.

    DeepSeek convention: weight_scale_inv stores the dequant scale (multiply by it
    to recover FP32). Block size is inferred from shape ratios — typically 128x128.
    <20>zExpected 2D weight, got shape r
<00>
zCannot align 1D scale_inv z to weight <20>
<01>dimN)r<00>shape<70>float<61>numel<65>	unsqueeze<7A>
ValueError<EFBFBD>repeat_interleave)	rr<00>
M<>
N<>sm<73>sn<73>bm<62>bn<62>
scale_fullr
r
r
<00>dequant_fp8_to_fp32Xs 











r))gg<00>?g<00>?g<00>?g@g@g@<40>@)
<01>dtype)g<00>?g<00>?g<00>?g<00>?g@g@<40>@r*g|@<40>
xc
CsP|d
k<00>t
j<02>
}
|<00><03>jtd<02>
}t<06>|j<07>
}t
<01>||<02>	<09><00><02>t
j<02>
}|
d>|BS)z<>Round x to nearest FP4 E2M1 representable, return 4-bit index in [0..15].

    Index encoding: bit 3 = sign, bits 0..2 = magnitude index into FP4_E2M1_VALUES.
    r
)
<01>max<61>)
<EFBFBD>to<74>torch<63>uint8<74>abs<62>clamp_<70>FP4_MAX<41>FP4_BOUNDARIES<45>device<63>searchsorted<65>
contiguous)r-<00>sign<67>abs_x<5F>
boundaries<EFBFBD>mag_idxr
r
r
<00>round_to_fp4_e2m1_index<65>s



r><00>x_fp32<33>scale_2cCs<>|j\}}|d
dkrt
d|j<00><00><02>
<01>
|<00>||d
d
<01>}|<04><03>jdd<05>
}|t|
}|jdd<07>
}|<06>tj	<09>
}|
|<07>
<EFBFBD>}||<08>d<04>
jdd<07>
}	t|	<09>
}
|
<EFBFBD>||<03><02>
<0A>}
|
dd<08>ddd	<09>f}|
dd<08>d
dd	<09>f}||d>B<00>tj<0E>
}
|
|fS)aN
Quantize an FP32 weight to NVFP4 given a (possibly joint) global scale.

    Args:
        x_fp32: [M, N] FP32 tensor, N must be divisible by 16
        scale_2: scalar FP32 tensor

    Returns:
        packed: [M, N//2] uint8, two FP4 values per byte (low nibble first)
        weight_scale: [M, N//16] FP8 E4M3 per-block scales
    <20>r
z NVFP4 requires N % 16 == 0; got <20><><EFBFBD><EFBFBD><EFBFBD>r<00><><EFBFBD><EFBFBD><EFBFBD>KH<4B>9<EFBFBD>
<01>minNrr<00>)rr <00>viewr3<00>amaxr5r4r0r1<00>
float8_e4m3fnrrr>r9r2)r?r@r"r#<00>blocks<6B>
block_amax<EFBFBD>block_scale_fp32<33>block_scale_fp8<70>	effective<76>scaled<65>fp4_idx<64>low<6F>high<67>packedr
r
r
<00>quantize_to_nvfp4<70>s 








rT<00>tensors_fp32cGs4t<00>
d
d<02>|D<00>
<01>
<01><02>}
|
tt}|jdd<04>
<01><06>S)zwCompute joint NVFP4 global scale_2 across one or more FP32 tensors.

    scale_2 = amax / (FP4_MAX * FP8_E4M3_MAX)
    c
Ssg|]}
|
<01><00><00>
<01><00>qSr
)r3r.)r<00>
tr
r
r
<00>
<listcomp><3E>sz(compute_global_scale.<locals>.<listcomp>rCrD)r1<00>stackr.r5<00>FP8_E4M3_MAXr4r)rUrHr@r
r
r
<00>compute_global_scale<6C>s
rZc@sFeZ
dZd
Zdedefdd<05>Zdd<07>Zded	e	j
fd
d<0B>Zdd
<0A>ZdS)<0F>ShardedSafetensorsWriterzJWrites tensors to a sequence of safetensors shards, building an index map.<2E>out_dir<69>max_shard_bytescCs>|
|_|jj
d
d
d<02>
||_i|_d|_d|_i|_g|_dS)NT<4E><02>parents<74>exist_okr
)r\<00>mkdirr]<00>current<6E>
current_bytes<65>	shard_idx<64>
weight_map<EFBFBD>shard_filenames)<03>selfr\r]r
r
r
<00>__init__<5F>s








z!ShardedSafetensorsWriter.__init__c
Csx|jsdS|j
d
7_
d|j
d<03>d<04>}
|j|
}t|jt|<02>
<01>
|jD]}|
|j|<q$|j<06>|
<01>

|j<00><08>
d|_	dS)Nr<00>model-<2D>05dz-of-PLACEHOLDER.safetensorsr
)
rbrdr\r<00>strrerf<00>append<6E>clearrc)rg<00>fname<6D>pathrr
r
r
<00>_flush<73>s













zShardedSafetensorsWriter._flushr<00>tensorcCsZ|<02><00><00>
<01><00><02>}|<03><03>|<03><04>}|jr|j||jkr|<00><08>
||j|
<|j|7_dSr)	<09>detach<63>cpur9r<00>element_sizerbrcr]rp)rgrrqrV<00>sizer
r
r
<00>add<64>s





zShardedSafetensorsWriter.addc
s<>|<00><00>
t
|j<02>
}
i<00>|jD]%}t|<02>d
<01>
d<00>
}d|d<04>d|
d<04>d<06>}|j|<00>|j|<00>

|<04>|<q<0E>f
dd<08>|j<07><08>D<00>
|_|jS)	N<>
-rrirjz-of-<2D>.safetensorsc
si|]	\}
}|
<01>|<00>qSr
r
)r<00>
k<>
v<>
<01>new_mapr
r
<00>
<dictcomp><3E><00>z2ShardedSafetensorsWriter.close.<locals>.<dictcomp>)	rp<00>lenrf<00>int<6E>splitr\<00>renamere<00>items)rg<00>total<61>	old_fname<6D>idx<64>	new_fnamer
r{r
<00>close<73>s










zShardedSafetensorsWriter.closeN)
<0A>__name__<5F>
__module__<EFBFBD>__qualname__<5F>__doc__rr<>rhrprkr1<00>Tensorrvr<>r
r
r
r
r[<00>s

	r[<00>src_dirc
Cs<>
t|d
<00>
<01>
}
t
<01>|
<01>
}Wd<00>
n1sw



Y
|d}tt<04>
}|<03><05>D]\}}||<00>|<05>

q'dd<05>|D<00>
}tt<07>
}|D]}	t<08>	|	<09>
}
|
rZ|
<EFBFBD>
d<06>
|
<EFBFBD>
d<07>
}}|	|||<q@t<0B>}
g}|<08><05>D]&\}}d|vr<>d	|vr<>|<0E>||d|d	f<03>

|
<0A>|d<00>

|
<0A>|d	<00>

qdg}g}g}|D]4}	|	<09>
d
<EFBFBD>
r<>|	<09>d
d<0B>|vr<>|<12>|	<09>

q<>|	|
vr<>q<EFBFBD>t|	<09>
r<>|<11>|	<09>

q<>|	<09>
d<0B>
r<>|<10>|	<09>

q<>|<11>|	<09>

q<>|t|<04>
||||d<0C>S)
a<>
Build the conversion plan from index.json.

    Returns:
        weight_map: name -> shard filename
        shard_to_names: shard filename -> list of names in that shard
        expert_pair_groups: list of (group_name, name_w1, name_w3)
            For each expert, the gate_proj/up_proj pair gets a shared scale_2.
        solo_quantize: list of names to quantize independently
        preserve: list of names to copy unchanged
    <20>model.safetensors.index.jsonNrec
Ssg|]	}
|
<01>d<00>
r|
<01>qS)
<01>.weight)
<01>endswith)r<00>
nr
r
r
rW
r~zbuild_plan.<locals>.<listcomp>rr<00>	gate_proj<6F>up_proj<6F>.scaler<65>)re<00>shard_to_names<65>expert_pair_groups<70>
solo_quantize<7A>preserve<76>scale_companions)<10>open<65>json<6F>loadr<00>listr<74>rl<00>dict<63>EXPERT_PAIR_REr<00>group<75>setrvr<><00>replacer)r<><00>
f<>indexrer<>r<00>fn<66>all_weights<74>expert_pairsr<73><00>
m<>base<73>kind<6E>paired_namesr<73><00>partsr<73>r<>r<>r
r
r
<00>
build_plan
sZ
<1C>







<02>





<02>


















<06>r<EFBFBD>c@s:eZ
dZd
Zd
dedefdd<06>Zdefdd	<09>Zd
d<0B>Z	dS)<0E>
ShardCachezELazy per-shard safe_open cache so we don't re-open shards repeatedly.rFr<><00>max_opencCs|
|_||_
i|_dSr)r<>r<><00>handles)rgr<>r<>r
r
r
rhP
s



zShardCache.__init__<5F>shard_fnamecCsv|
|jvr
|j|
St
|j<00>
|jkr'tt|j<00>
<01>
}|j|<00>ddd<00>
|j|=t|j|
d
d<02>}|<03><08>
||j|
<|S)N<>pt)
<01>	framework)	r<>rr<><00>next<78>iter<65>__exit__rr<><00>	__enter__)rgr<><00>old_fn<66>
hr
r
r
<00>getU
s











zShardCache.getc
Cs,|j<00>
<01>D]	}
|
<01>ddd<00>
q|j<00><03>
dSr)r<><00>valuesr<73>rm)rgr<>r
r
r
r<>b
s


zShardCache.closeN)
rF)
r<EFBFBD>r<>r<>r<>rr<>rhrkr<>r<>r
r
r
r
r<>M
s


r<><00>cachecCsl|<00>|
|<00>
<01>
|<02>
}|<02>d
d<02>}d}||
vr2z|<00>|
|<00>
<01>
|<04>
}W||fSty1


Y||fSw||fS)z5Load an FP8 weight with its scale companion (if any).r<>r<>N)r<><00>
get_tensorr<EFBFBD><00>	Exception)r<>rer<00>weight<68>
scale_name<EFBFBD>scaler
r
r
<00>load_weight_and_scaleh
s




<0C>
<02>r<>c+
st<00>
<01>}|jd
ddd<04>
|jdddd<04>
|jdtdd	d
<EFBFBD>
|jdtdd
<0A>
|jdtddd
<EFBFBD>
|jddd<13>
|<00><05>}
t|
j<07>
<01><08>}t|
j	<09>
<01><08><00>|d<00>
<EFBFBD>sUt<0B>d|<02><00><02>

t
<0A>|
jdkrgt
j<10><11>rgd|
j<0F><00>n
d<17>
}td|<03><00><02>

t<13>|<03>
atd<19>

t|<02>
}t|d<00>
}t|d<00>
}t|d<00>
}t|d<00>
}td|d<1F><04><02>

td |d<1F><04><02>

td!|d<1F><04><02>

td"|d<1F><04><02>

|
jr<>td#<23>

dS<00>jddd$<24>
t|d%d&<26>}	t<1A>t|
jd'<00>
d(<28>}
|d)}t<1C><1C>}t|dd*d+d,<2C>D]}
|	<09>||
<00>
<01>|
<0A>
}|
<EFBFBD> |
|<0E>
q<>t|dd-d.d,<2C>D]<5D>\}}}t!|	||<10>\}}t!|	||<11>\}}t
<0A>"<22><00>}
|du
<01>
r.t#|<12>|<03>
|<13>|<03>
<01>n|<12><04><00>|<03>
}|du
<01>
rEt#|<14>|<03>
|<15>|<03>
<01>n|<14><04><00>|<03>
}t$||<17>}t%||<18>\}}t%||<18>\}}|
<EFBFBD> ||<19>
|
<EFBFBD> |<10>&d/d0<64>|<1A>
|
<EFBFBD> |<10>&d/d1<64>|<18>
|
<EFBFBD> ||<1B>
|
<EFBFBD> |<11>&d/d0<64>|<1C>
|
<EFBFBD> |<11>&d/d1<64>|<18>
Wd<00>
n	1<00>
s<>w



Y
<00>
qt|dd2d+d,<2C>D]]}
t!|	||
<0A>\}}t
<0A>"<22><00>E
|du
<01>
r<>t#|<1D>|<03>
|<1E>|<03>
<01>}n|<1D><04><00>|<03>
}t$|<1F>
}t%||<18>\} }!|
<EFBFBD> |
| <20>
|
<EFBFBD> |
<0A>&d/d0<64>|!<21>
|
<EFBFBD> |
<0A>&d/d1<64>|<18>
Wd<00>
n	1<00>sw



Y
<00>
q<>|
<EFBFBD>'<27>}"|	<09>'<27>
t(<28>f
d3d4<64>t)|"<22>*<2A><00>
D<00>
<01>
}#d5|#i
|"d6<64>}$t+<2B>dd7<64><02>}%t,j-|$|%d8d9<64>
Wd<00>
n	1<00>sEw



Y
|<02>.<2E>D]3}&|&<26>/<2F><00>rg<72>|&j0}'|'<27>
<EFBFBD><00>set1<74>2|&|'<27>
<00>qN|&j3d:k<02>ro<72>qN|&j0dk<02>rw<72>qNt1<74>4|&<26>|&j0<00>
<00>qN<71>d;}(|(<28>
<EFBFBD><00>r<>t+|(<28>
<01>
}%t,<2C>5|%<25>
})Wd<00>
n	1<00>s<>w



Y
d<d=d>d?g
dd@dAdBddC<64>dD<64>i
t6dE<64>|)dF<t+|(d7<64><02>}%t,j-|)|%d8d9<64>
Wd<00>
n	1<00>s<>w



Y
t<1C><1C>|}*tdG|*dHdI<64>dJ<64><03>

tdK<64><00><00><02>

tdL|#d'dM<64>dNtt)|"<22>*<2A><00>
<01>
<01>dO<64><05>

dS)PNz--srcTzSource FP8 model directory)<02>required<65>helpz--dstzOutput NVFP4 model directoryz--gpur
zCUDA device, -1 for CPU)<03>type<70>defaultr<74>z--shard-size-gbr,)r<>r<>z	--workersrFzoConcurrent tensor-conversion workers (lots of small tensors benefit; actual GPU compute is serialized by torch)z	--dry-run<75>
store_true)
<01>actionr<6E>zNo index.json at zcuda:rszCompute device: zBuilding conversion plan...r<>r<>r<>r<>z&  Expert pair groups (joint scale_2): <20>
,z&  Solo quantize tensors:              z&  Preserved tensors:                  z&  Scale companions consumed:          u'
Dry run — exiting before any writes.r^<00>)
r<>i@)
r]re<00>Preserverq)<02>desc<73>unitzExpert pairs<72>pairr<72>z
.weight_scalez.weight_scale_2z
Solo quantizec
3s<00>|]
}
<01>|
<00><00>j
V
qdSr)<02>stat<61>st_size)rr<><00>
<01>dstr
r
r<00>
s<02>

<EFBFBD>zmain.<locals>.<genexpr><3E>
total_size)<02>metadatare<00>
wr)
<01>indentrxzconfig.jsonzcompressed-tensorsznvfp4-pack-quantized<65>group_0<5F>Linearr<00>tensor_grouprA)<05>num_bitsr<73><00>strategy<67>
group_size<EFBFBD>	symmetric)<02>targets<74>weights)<04>quant_method<6F>format<61>
config_groups<70>ignore<72>quantization_configz	
Done in iz.2fr<EFBFBD>zOutput: zTotal size: z.1fz GB across z shards)7<>argparse<73>ArgumentParser<65>add_argumentr<74>r<00>
parse_argsr<00>src<72>resolver<65><00>exists<74>sys<79>exitr1r7<00>gpu<70>cuda<64>is_available<6C>printr6r0r<>r<00>dry_runrar<>r[<00>
shard_size_gb<67>timerr<>r<>rvr<><00>no_gradr)rZrTr<>r<><00>sumr<6D>r<>r<>r<><00>dump<6D>iterdir<69>is_dirr<00>shutil<69>copytree<65>suffix<69>copy2r<32><00>PRESERVE_REGEXES)+<2B>ap<61>argsr<73>r7<00>plan<61>n_pairs<72>n_solo<6C>
n_preserve<EFBFBD>n_scalesr<73><00>writerre<00>t_startrrVr<><00>name_w1<77>name_w3<77>w1_fp8<70>s1<73>w3_fp8<70>s3<73>w1<77>w3r@<00>packed1<64>blk1<6B>packed3<64>blk3<6B>w_fp8<70>
sr<73>rS<00>blk<6C>final_weight_mapr<70><00>	new_indexr<78>rn<00>dst_sub<75>cfg_path<74>cfg<66>elapsedr
r<>r
<00>mainz
s<>







<06>



*




















.
.






<1E><04>









<1E><04>


<08>
<06>
<1E>
















<1E>





<04><04><02>
<EFBFBD>
<1E>

,
r
<00>__main__)-r<>r<>r<><00>rer<65>r<>r<><00>collectionsr<00>concurrent.futuresr<00>pathlibrr1<00>safetensorsr<00>safetensors.torchrrr<><00>compile<6C>joinrr<>rkrrr<>r)rq<00>float32<33>FP4_E2M1_VALUESr6r5rYr><00>tuplerTrZr[r<>r<>r<>r
r<>r
r
r
r
<00><module>
sZ
#











"

<06>

<06>


<02><02>
<EFBFBD>,8H"

<04>