Required Instance Propertyswift-transformers 0.1.17Tokenizers
eosToken
Tokenizer.swift:40var eosToken: String? { get }
var eosToken: String? { get }
import Tokenizers
protocol TokenizingModel
@frozen struct String
A Unicode string value that is a collection of characters.
var bosToken: String? { get }
var bosTokenId: Int? { get }
var eosTokenId: Int? { get }
var fuseUnknownTokens: Bool { get }
var unknownToken: String? { get }
var unknownTokenId: Int? { get }
func callAsFunction(_ text: String) -> [String]
func convertIdToToken(_ id: Int) -> String?
func convertIdsToTokens(_ ids: [Int]) -> [String?]
func convertTokenToId(_ token: String) -> Int?
func convertTokensToIds(_ tokens: [String]) -> [Int?]
func tokenize(text: String) -> [String]