Instance Method (Default implementation)swift-transformers 0.1.17Tokenizers

convertTokensToIds(_:)

Tokenizer.swift:64
func convertTokensToIds(_ tokens: [String]) -> [Int?]