verl/trainer/ppo/ray_trainer.py

• Role · class · L57-L67 — class Role(Enum)
• ResourcePoolManager · class · L71-L93 — class ResourcePoolManager
• create_resource_pool · method · L80-L89 — def create_resource_pool(self)
• get_resource_pool · method · L91-L93 — def get_resource_pool(self, role: Role) -> RayResourcePool
• apply_kl_penalty · function · L100-L129 — def apply_kl_penalty(data: DataProto, kl_ctrl: core_algos.AdaptiveKLController, kl_penalty='kl')
• compute_advantage · function · L131-L174 — def compute_advantage(data: DataProto, adv_estimator, gamma=1.0, lam=1.0, num_repeat=1): # prepare response group # TODO: add other ways to estimate advantages
• reduce_metrics · function · L177-L180 — def reduce_metrics(metrics: dict)
• _compute_response_info · function · L183-L196 — def _compute_response_info(batch)
• compute_data_metrics · function · L199-L301 — def compute_data_metrics(batch, use_critic=True): # TODO: add response length
• compute_timing_metrics · function · L304-L325 — def compute_timing_metrics(batch, timing_raw)
• _timer · function · L329-L332 — def _timer(name: str, timing_raw: Dict[str, float])
• RayPPOTrainer · class · L335-L1407 — class RayPPOTrainer(object)
• init · method · L342-L385 — def init(self, config, tokenizer, role_worker_mapping: dict[Role, WorkerType], resource_pool_manager: ResourcePoolManager, ray_worker_group_cls: RayWorkerGroup = RayWorkerGroup, reward_fn=None, val_reward_fn=None): # assert torch.cuda.is_available(), 'cuda must be available on driver'
• _init_logger · method · L387-L392 — def _init_logger(self)
• _create_dataloader · method · L394-L456 — def _create_dataloader(self)
• _validate · method · L458-L625 — def _validate(self)
• _merge_stats · function · L473-L483 — def _merge_stats(accum, new_stats)
• init_workers · method · L628-L699 — def init_workers(self)
• _save_checkpoint · method · L701-L713 — def _save_checkpoint(self)
• _balance_batch · method · L715-L730 — def _balance_batch(self, batch: DataProto, metrics, logging_prefix='global_seqlen')
• fit · method · L732-L972 — def fit(self)
• _save_sample_level_data · method · L976-L1190 — def _save_sample_level_data(self, batch: DataProto, gen_stats: dict, reward_stats: dict, split='train')
• _collect_and_log_statistics · method · L1192-L1390 — def _collect_and_log_statistics(self, batch, metrics: dict, split='train'): # 检查是否有统计信息（支持 DataProto 和 SimpleNamespace）
• _mean_list · function · L1330-L1334 — def _mean_list(key, default=0.0)
• _create_loss_mask · method · L1394-L1407 — def _create_loss_mask(self, batch, metrics)