Instance Propertyswift-transformers 0.1.17Tokenizers
bosTokenId
BertTokenizer.swift:22var bosTokenId: Int?
var bosTokenId: Int?
import Tokenizers
class BertTokenizer
@frozen struct Int
A signed integer value type.
required convenience init(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String : Int]) throws
init(vocab: [String : Int], merges: [String]?, tokenizeChineseChars: Bool = true, bosToken: String? = nil, eosToken: String? = nil, fuseUnknownTokens: Bool = false, doLowerCase: Bool = true)
var bosToken: String?
var eosToken: String?
var eosTokenId: Int?
let fuseUnknownTokens: Bool
var unknownToken: String? { get }
var unknownTokenId: Int? { get }
func convertIdToToken(_ id: Int) -> String?
func convertTokenToId(_ token: String) -> Int?
func tokenize(text: String) -> [String]