Required Instance Methodswift-transformers 0.1.17Tokenizers
applyChatTemplate(messages:)
The appropriate chat template is selected from the tokenizer config
func applyChatTemplate(messages: [Message]) throws -> [Int]
The appropriate chat template is selected from the tokenizer config
func applyChatTemplate(messages: [Message]) throws -> [Int]
s10Tokenizers9TokenizerP17applyChatTemplate8messagesSaySiGSaySDySSypGG_tKF
What are these?5Q27A
import Tokenizers
protocol Tokenizer
typealias Message = [String : Any]
@frozen struct Int
A signed integer value type.
var bosToken: String? { get }
var bosTokenId: Int? { get }
var eosToken: String? { get }
var eosTokenId: Int? { get }
var hasChatTemplate: Bool { get }
var unknownToken: String? { get }
var unknownTokenId: Int? { get }
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument) throws -> [Int]
The chat template is provided as a string literal or specified by name
func applyChatTemplate(messages: [Message], chatTemplate: String) throws -> [Int]
The chat template is provided as a string literal
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument?, addGenerationPrompt: Bool, truncation: Bool, maxLength: Int?, tools: [ToolSpec]?) throws -> [Int]
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument?, addGenerationPrompt: Bool, truncation: Bool, maxLength: Int?, tools: [ToolSpec]?, additionalContext: [String : Any]?) throws -> [Int]
func applyChatTemplate(messages: [Message], tools: [ToolSpec]?) throws -> [Int]
The appropriate chat template is selected from the tokenizer config
func applyChatTemplate(messages: [Message], tools: [ToolSpec]?, additionalContext: [String : Any]?) throws -> [Int]
The appropriate chat template is selected from the tokenizer config
func callAsFunction(_ text: String, addSpecialTokens: Bool) -> [Int]
func convertIdToToken(_ id: Int) -> String?
func convertIdsToTokens(_ ids: [Int]) -> [String?]
func convertTokenToId(_ token: String) -> Int?
func convertTokensToIds(_ tokens: [String]) -> [Int?]
func decode(tokens: [Int]) -> String
Decode
func decode(tokens: [Int], skipSpecialTokens: Bool) -> String
func encode(text: String) -> [Int]
Main entry point
func encode(text: String, addSpecialTokens: Bool) -> [Int]
func tokenize(text: String) -> [String]