NeMo Common Collection API

NeMo Common Collection API

#

The common collection contains things that could be used across all collections.

Callbacks

Exponential Moving Average (EMA)

Losses

AggregatorLoss

__init__()

forward()

input_types

output_types

CrossEntropyLoss

__init__()

forward()

input_types

output_types

MSELoss

__init__()

forward()

input_types

output_types

SmoothedCrossEntropyLoss

__init__()

forward()

input_types

output_types

SpanningLoss

__init__()

forward()

input_types

output_types

Metrics

Perplexity

compute()

full_state_update

update()

Tokenizers

AutoTokenizer

__init__()

add_special_tokens()

additional_special_tokens_ids

apply_chat_template()

bos_id

cls_id

eod

eos_id

ids_to_text()

ids_to_tokens()

inv_vocab

mask_id

name

pad_id

save_pretrained()

save_vocabulary()

sep_id

text_to_ids()

text_to_tokens()

token_to_id()

tokens_to_ids()

tokens_to_text()

unk_id

vocab

vocab_size

SentencePieceTokenizer

__init__()

add_special_tokens()

additional_special_tokens_ids

bos_id

cls_id

eos_id

get_var_bpe_extension()

ids_to_text()

ids_to_tokens()

mask_id

pad_id

sep_id

text_to_ids()

text_to_ids_var_bpe()

text_to_tokens()

token_to_id()

tokens_to_ids()

tokens_to_text()

unk_id

vocab

TokenizerSpec

add_special_tokens()

apply_chat_template()

bos

cls

eod

eos

ids_to_text()

ids_to_tokens()

mask

name

pad

sep

text_to_ids()

text_to_ids_var_bpe()

text_to_tokens()

tokens_to_ids()

tokens_to_text()

unique_identifiers

Data

ConcatDataset

get_iterable()

random_generator()

round_robin_generator()

temperature_generator()

ConcatMapDataset

S3 Checkpointing

S3CheckpointIO

S3Utils and Dependencies

s3_dirpath_utils

S3 Demands and ExpManager Details When Running at Scale