Required Instance Methodswift-transformers 0.1.17Tokenizers
convertTokensToIds(_:)
Tokenizer.swift:135func convertTokensToIds(_ tokens: [String]) -> [Int?]
func convertTokensToIds(_ tokens: [String]) -> [Int?]
s10Tokenizers9TokenizerP18convertTokensToIdsySaySiSgGSaySSGF
What are these?5BS37
import Tokenizers
protocol Tokenizer
@frozen struct String
A Unicode string value that is a collection of characters.
@frozen struct Int
A signed integer value type.
var bosToken: String? { get }
var bosTokenId: Int? { get }
var eosToken: String? { get }
var eosTokenId: Int? { get }
var hasChatTemplate: Bool { get }
var unknownToken: String? { get }
var unknownTokenId: Int? { get }
func applyChatTemplate(messages: [Message]) throws -> [Int]
The appropriate chat template is selected from the tokenizer config
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument) throws -> [Int]
The chat template is provided as a string literal or specified by name
func applyChatTemplate(messages: [Message], chatTemplate: String) throws -> [Int]
The chat template is provided as a string literal
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument?, addGenerationPrompt: Bool, truncation: Bool, maxLength: Int?, tools: [ToolSpec]?) throws -> [Int]
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument?, addGenerationPrompt: Bool, truncation: Bool, maxLength: Int?, tools: [ToolSpec]?, additionalContext: [String : Any]?) throws -> [Int]
func applyChatTemplate(messages: [Message], tools: [ToolSpec]?) throws -> [Int]
The appropriate chat template is selected from the tokenizer config
func applyChatTemplate(messages: [Message], tools: [ToolSpec]?, additionalContext: [String : Any]?) throws -> [Int]
The appropriate chat template is selected from the tokenizer config
func callAsFunction(_ text: String, addSpecialTokens: Bool) -> [Int]
func convertIdToToken(_ id: Int) -> String?
func convertIdsToTokens(_ ids: [Int]) -> [String?]
func convertTokenToId(_ token: String) -> Int?
func decode(tokens: [Int]) -> String
Decode
func decode(tokens: [Int], skipSpecialTokens: Bool) -> String
func encode(text: String) -> [Int]
Main entry point
func encode(text: String, addSpecialTokens: Bool) -> [Int]
func tokenize(text: String) -> [String]
func convertTokensToIds(_ tokens: [String]) -> [Int?]