Instance Propertyswift-transformers 0.1.17Tokenizers
unknownToken
Tokenizer.swift:244var unknownToken: String? { get }var unknownToken: String? { get }import Tokenizersclass PreTrainedTokenizer@frozen struct StringA Unicode string value that is a collection of characters.
required init(tokenizerConfig: Config, tokenizerData: Config) throws var bosToken: String? { get }var bosTokenId: Int? { get }var eosToken: String? { get }var eosTokenId: Int? { get }var fuseUnknownTokens: Bool { get }var hasChatTemplate: Bool { get }var unknownTokenId: Int? { get }func applyChatTemplate(messages: [Message]) throws -> [Int] func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument) throws -> [Int] func applyChatTemplate(messages: [Message], chatTemplate: String) throws -> [Int] func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument? = nil, addGenerationPrompt: Bool = false, truncation: Bool = false, maxLength: Int? = nil, tools: [ToolSpec]? = nil) throws -> [Int] func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument? = nil, addGenerationPrompt: Bool = false, truncation: Bool = false, maxLength: Int? = nil, tools: [ToolSpec]? = nil, additionalContext: [String : Any]? = nil) throws -> [Int] func applyChatTemplate(messages: [Message], tools: [ToolSpec]? = nil) throws -> [Int] func applyChatTemplate(messages: [Message], tools: [ToolSpec]? = nil, additionalContext: [String : Any]? = nil) throws -> [Int] func convertIdToToken(_ id: Int) -> String? func convertTokenToId(_ token: String) -> Int? func decode(tokens: [Int], skipSpecialTokens: Bool = false) -> String func encode(text: String) -> [Int] func encode(text: String, addSpecialTokens: Bool = true) -> [Int] Main entry point
func tokenize(text: String) -> [String]