zsfm_nn/ffn.rs
1use anyhow::Result;
2use candle_core::Tensor;
3
4use crate::linear::linear_nobias;
5
6/// SwiGLU feed-forward: `w2(silu(w1(x)) * w3(x))`, all projections bias-free.
7pub fn swiglu_ffn(x: &Tensor, fc1_w: &Tensor, fc2_w: &Tensor, gate_w: &Tensor) -> Result<Tensor> {
8 let content = linear_nobias(x, fc1_w)?.silu()?;
9 let gate = linear_nobias(x, gate_w)?;
10 let h = (content * gate)?;
11 linear_nobias(&h, fc2_w)
12}