verl/workers/rollout/tokenizer.py

• HybridEngineBaseTokenizer · class · L23-L162 — class HybridEngineBaseTokenizer(ABC)
• vocab_size · method · L28-L32 — def vocab_size(self)
• pad_token_id · method · L36-L40 — def pad_token_id(self)
• eos_token_id · method · L44-L49 — def eos_token_id(self)
• all_special_ids · method · L53-L57 — def all_special_ids(self) -> List[int]
• all_special_tokens · method · L61-L67 — def all_special_tokens(self) -> List[str]
• encode · method · L70-L83 — def encode(self, text)
• decode · method · L86-L113 — def decode( self, token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"], skip_special_tokens: bool = False, clean_up_tokenization_spaces: bool = None, **kwargs, ) -> str
• convert_ids_to_tokens · method · L116-L132 — def convert_ids_to_tokens(self, ids: Union[int, List[int]], skip_special_tokens: bool = False) -> Union[str, List[str]]
• get_added_vocab · method · L135-L144 — def get_added_vocab(self) -> Dict[str, int]
• convert_tokens_to_string · method · L147-L158 — def convert_tokens_to_string(self, tokens: List[str]) -> str
• is_fast · method · L161-L162 — def is_fast(self)