Instance Methodswift-transformers 0.1.17Tokenizers
decode(tokens:skipSpecialTokens:)
Tokenizer.swift:382func decode(tokens: [Int], skipSpecialTokens: Bool = false) -> String
func decode(tokens: [Int], skipSpecialTokens: Bool = false) -> String
s10Tokenizers19PreTrainedTokenizerC6decode6tokens17skipSpecialTokensSSSaySiG_SbtF
What are these?179D4
import Tokenizers
class PreTrainedTokenizer
@frozen struct Int
A signed integer value type.
@frozen struct Bool
A value type whose instances are either true
or false
.
@frozen struct String
A Unicode string value that is a collection of characters.
required init(tokenizerConfig: Config, tokenizerData: Config) throws
var bosToken: String? { get }
var bosTokenId: Int? { get }
var eosToken: String? { get }
var eosTokenId: Int? { get }
var fuseUnknownTokens: Bool { get }
var hasChatTemplate: Bool { get }
var unknownToken: String? { get }
var unknownTokenId: Int? { get }
func applyChatTemplate(messages: [Message]) throws -> [Int]
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument) throws -> [Int]
func applyChatTemplate(messages: [Message], chatTemplate: String) throws -> [Int]
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument? = nil, addGenerationPrompt: Bool = false, truncation: Bool = false, maxLength: Int? = nil, tools: [ToolSpec]? = nil) throws -> [Int]
func applyChatTemplate(messages: [Message], chatTemplate: ChatTemplateArgument? = nil, addGenerationPrompt: Bool = false, truncation: Bool = false, maxLength: Int? = nil, tools: [ToolSpec]? = nil, additionalContext: [String : Any]? = nil) throws -> [Int]
func applyChatTemplate(messages: [Message], tools: [ToolSpec]? = nil) throws -> [Int]
func applyChatTemplate(messages: [Message], tools: [ToolSpec]? = nil, additionalContext: [String : Any]? = nil) throws -> [Int]
func convertIdToToken(_ id: Int) -> String?
func convertTokenToId(_ token: String) -> Int?
func encode(text: String) -> [Int]
func encode(text: String, addSpecialTokens: Bool = true) -> [Int]
Main entry point
func tokenize(text: String) -> [String]