struct AutoTokenizer
s10Tokenizers13AutoTokenizerV
2V571
import Tokenizers
static func from(modelFolder: URL, hubApi: HubApi = .shared) async throws -> Tokenizer
static func from(pretrained model: String, hubApi: HubApi = .shared) async throws -> Tokenizer
static func from(tokenizerConfig: Config, tokenizerData: Config) throws -> Tokenizer