Required Initializerswift-transformers 0.1.17Tokenizers
init(config:)
PreTokenizer.swift:23init(config: Config)
init(config: Config)
import Tokenizers
protocol PreTokenizer
@dynamicMemberLookup struct Config
func callAsFunction(text: String, options: PreTokenizerOptions) -> [String]
func callAsFunction(texts: [String], options: PreTokenizerOptions) -> [String]
func preTokenize(text: String, options: PreTokenizerOptions) -> [String]
func preTokenize(texts: [String], options: PreTokenizerOptions) -> [String]