zsfm_moment/
config.rs

1/// MOMENT-1-large model configuration (T5-large backbone).
2#[derive(Clone)]
3pub struct MomentConfig {
4    pub d_model: usize,         // 1024
5    pub n_layers: usize,        // 24
6    pub n_heads: usize,         // 16
7    pub head_dim: usize,        // 64  (d_kv)
8    pub d_ff: usize,            // 2816
9    pub seq_len: usize,         // 512 (max context)
10    pub patch_len: usize,       // 8
11    pub patch_stride: usize,    // 8
12    pub num_patches: usize,     // 64  (seq_len / patch_stride)
13    pub rel_attn_num_buckets: usize,    // 32
14    pub rel_attn_max_distance: usize,   // 128
15    pub layer_norm_eps: f64,    // 1e-6
16}
17
18impl MomentConfig {
19    pub fn default() -> Self {
20        let seq_len = 512;
21        let patch_len = 8;
22        let patch_stride = 8;
23        Self {
24            d_model: 1024,
25            n_layers: 24,
26            n_heads: 16,
27            head_dim: 64,
28            d_ff: 2816,
29            seq_len,
30            patch_len,
31            patch_stride,
32            num_patches: seq_len / patch_stride, // 64
33            rel_attn_num_buckets: 32,
34            rel_attn_max_distance: 128,
35            layer_norm_eps: 1e-6,
36        }
37    }
38}
zsfm_moment/config.rs

zsfm_moment/
config.rs
