Instance Propertyswift-transformers 0.1.17Models
tokenizer
LanguageModel.swift:190var tokenizer: Tokenizer { get async throws }
var tokenizer: Tokenizer { get async throws }
s6Models13LanguageModelC9tokenizer10Tokenizers9Tokenizer_pvp
What are these?615G1
import Models
class LanguageModel
protocol Tokenizer
required init(model: MLModel)
static func loadCompiled(url: URL, computeUnits: MLComputeUnits = .cpuAndGPU) throws -> LanguageModel
var bosTokenId: Int? { get async throws }
var defaultDoSample: Bool { get async throws }
var defaultGenerationConfig: GenerationConfig { get }
var description: String { get }
var eosTokenId: Int? { get async throws }
var inputIdsDescription: MLFeatureDescription { get }
var inputIdsName: String { get }
var inputIdsShape: [Int] { get }
The expected shape of the models latent sample input
let maxContextLength: Int
let minContextLength: Int
let model: MLModel
var modelConfig: Config { get async throws }
var modelName: String { get }
var modelType: String? { get async throws }
var requiresAttention: Bool { get }
var textGenerationParameters: Config? { get async throws }
var tokenizerConfig: Config? { get async throws }
var tokenizerData: Config { get async throws }
func predictNextTokenScores(_ tokens: InputTokens, config: GenerationConfig) -> any MLShapedArrayProtocol