Research
High-velocity research, from experimental prototypes to controlled testing at scale.
- Reinforcement learning
- Reward modelling
- Post-training
- RL environments
- Long-horizon RL
- Evals
AI
API
UI
AI
- def group_advantages(rewards, eps=1e-4):
- # advantage is the reward’s z-score within its group
- mean = rewards.mean(dim=-1, keepdim=True)
- std = rewards.std(dim=-1, keepdim=True)
- return (rewards – mean) / (std + eps)
- def policy_loss(logprobs, old_logprobs, advantages, mask, clip=0.2):
- ratio = (logprobs – old_logprobs).exp()
- unclipped = ratio * advantages
- clipped = ratio.clamp(1 – clip, 1 + clip) * advantages
- per_token = -torch.min(unclipped, clipped)
- return (per_token * mask).sum() / mask.sum().clamp(min=1)
- def kl_to_ref(logprobs, ref_logprobs, mask):
- return (((ref_logprobs – logprobs).exp() – 1) * mask).sum() / mask.sum()
API
- fn select_replica(replicas: &[Replica], req: &Request) -> Option<usize> {
- let candidate = replicas
- .iter()
- .enumerate()
- .filter(|(_, r)| r.healthy && r.queue_depth < r.max_queue)
- .min_by_key(|(_, r)| {
- let prefill = r.prefill_cost(req.prompt_tokens);
- let reuse = r.prefix_match(&req.prefix_hash);
- projected_ttft(r.queue_depth, prefill, reuse)
- })
- .map(|(i, _)| i);
- // no healthy replica under queue limit: shed rather than queue
- candidate.or_else(|| fallback_replica(replicas, req))
- }
UI
- export const load = async () => {
- const res = await fetch(“/api/runs?status=active”);
- return { runs: await res.json() };
- };
- export const curves = (run) =>
- run.steps.map((s) => ({
- step: s.global_step,
- reward: s.reward_mean,
- kl: s.kl_to_ref,
- entropy: s.entropy,
- }));
- export const diverged = (run) =>
- curves(run).some((p) => !Number.isFinite(p.reward));
